* [Qemu-devel] [PATCH v12 16/22] vfio_pci: Updated to use vfio_set_irqs_validate_and_prepare()
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Updated vfio_pci.c file to use vfio_set_irqs_validate_and_prepare()
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I9f3daba89d8dba5cb5b01a8cff420412f30686c7
---
drivers/vfio/pci/vfio_pci.c | 34 +++++++---------------------------
1 file changed, 7 insertions(+), 27 deletions(-)
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 03b5434f4d5b..dcd7c2a99618 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -818,45 +818,25 @@ static long vfio_pci_ioctl(void *device_data,
} else if (cmd == VFIO_DEVICE_SET_IRQS) {
struct vfio_irq_set hdr;
- size_t size;
u8 *data = NULL;
int max, ret = 0;
+ size_t data_size = 0;
minsz = offsetofend(struct vfio_irq_set, count);
if (copy_from_user(&hdr, (void __user *)arg, minsz))
return -EFAULT;
- if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS ||
- hdr.count >= (U32_MAX - hdr.start) ||
- hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
- VFIO_IRQ_SET_ACTION_TYPE_MASK))
- return -EINVAL;
-
max = vfio_pci_get_irq_count(vdev, hdr.index);
- if (hdr.start >= max || hdr.start + hdr.count > max)
- return -EINVAL;
- switch (hdr.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
- case VFIO_IRQ_SET_DATA_NONE:
- size = 0;
- break;
- case VFIO_IRQ_SET_DATA_BOOL:
- size = sizeof(uint8_t);
- break;
- case VFIO_IRQ_SET_DATA_EVENTFD:
- size = sizeof(int32_t);
- break;
- default:
- return -EINVAL;
- }
-
- if (size) {
- if (hdr.argsz - minsz < hdr.count * size)
- return -EINVAL;
+ ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+ VFIO_PCI_NUM_IRQS, &data_size);
+ if (ret)
+ return ret;
+ if (data_size) {
data = memdup_user((void __user *)(arg + minsz),
- hdr.count * size);
+ data_size);
if (IS_ERR(data))
return PTR_ERR(data);
}
--
2.7.0
^ permalink raw reply related
* [PATCH v12 13/22] vfio: Introduce common function to add capabilities
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Vendor driver using mediated device framework should use
vfio_info_add_capability() to add capabilities.
Introduced this function to reduce code duplication in vendor drivers.
vfio_info_cap_shift() manipulated a data buffer to add an offset to each
element in a chain. This data buffer is documented in a uapi header.
Changing vfio_info_cap_shift symbol to be available to all drivers.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I6fca329fa2291f37a2c859d0bc97574d9e2ce1a6
---
drivers/vfio/vfio.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++-
include/linux/vfio.h | 3 +++
2 files changed, 62 insertions(+), 1 deletion(-)
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 388a3cbddbd9..3648fd74aa30 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -1797,8 +1797,66 @@ void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
for (tmp = caps->buf; tmp->next; tmp = (void *)tmp + tmp->next - offset)
tmp->next += offset;
}
-EXPORT_SYMBOL_GPL(vfio_info_cap_shift);
+EXPORT_SYMBOL(vfio_info_cap_shift);
+static int sparse_mmap_cap(struct vfio_info_cap *caps, void *cap_type)
+{
+ struct vfio_info_cap_header *header;
+ struct vfio_region_info_cap_sparse_mmap *sparse_cap, *sparse = cap_type;
+ size_t size;
+
+ size = sizeof(*sparse) + sparse->nr_areas * sizeof(*sparse->areas);
+ header = vfio_info_cap_add(caps, size,
+ VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1);
+ if (IS_ERR(header))
+ return PTR_ERR(header);
+
+ sparse_cap = container_of(header,
+ struct vfio_region_info_cap_sparse_mmap, header);
+ sparse_cap->nr_areas = sparse->nr_areas;
+ memcpy(sparse_cap->areas, sparse->areas,
+ sparse->nr_areas * sizeof(*sparse->areas));
+ return 0;
+}
+
+static int region_type_cap(struct vfio_info_cap *caps, void *cap_type)
+{
+ struct vfio_info_cap_header *header;
+ struct vfio_region_info_cap_type *type_cap, *cap = cap_type;
+
+ header = vfio_info_cap_add(caps, sizeof(*cap),
+ VFIO_REGION_INFO_CAP_TYPE, 1);
+ if (IS_ERR(header))
+ return PTR_ERR(header);
+
+ type_cap = container_of(header, struct vfio_region_info_cap_type,
+ header);
+ type_cap->type = cap->type;
+ type_cap->subtype = cap->subtype;
+ return 0;
+}
+
+int vfio_info_add_capability(struct vfio_info_cap *caps, int cap_type_id,
+ void *cap_type)
+{
+ int ret = -EINVAL;
+
+ if (!cap_type)
+ return 0;
+
+ switch (cap_type_id) {
+ case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
+ ret = sparse_mmap_cap(caps, cap_type);
+ break;
+
+ case VFIO_REGION_INFO_CAP_TYPE:
+ ret = region_type_cap(caps, cap_type);
+ break;
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL(vfio_info_add_capability);
/*
* Pin a set of guest PFNs and return their associated host PFNs for local
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 123e089388e9..b1aaf69cb470 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -111,6 +111,9 @@ extern struct vfio_info_cap_header *vfio_info_cap_add(
struct vfio_info_cap *caps, size_t size, u16 id, u16 version);
extern void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset);
+extern int vfio_info_add_capability(struct vfio_info_cap *caps,
+ int cap_type_id, void *cap_type);
+
struct pci_dev;
#ifdef CONFIG_EEH
extern void vfio_spapr_pci_eeh_open(struct pci_dev *pdev);
--
2.7.0
^ permalink raw reply related
* [PATCH v12 12/22] vfio: Add notifier callback to parent's ops structure of mdev
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Add a notifier calback to parent's ops structure of mdev device so that per
device notifer for vfio module is registered through vfio_mdev module.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: Iafa6f1721aecdd6e50eb93b153b5621e6d29b637
---
drivers/vfio/mdev/vfio_mdev.c | 19 +++++++++++++++++++
include/linux/mdev.h | 9 +++++++++
2 files changed, 28 insertions(+)
diff --git a/drivers/vfio/mdev/vfio_mdev.c b/drivers/vfio/mdev/vfio_mdev.c
index ffc36758cb84..1694b1635607 100644
--- a/drivers/vfio/mdev/vfio_mdev.c
+++ b/drivers/vfio/mdev/vfio_mdev.c
@@ -24,6 +24,15 @@
#define DRIVER_AUTHOR "NVIDIA Corporation"
#define DRIVER_DESC "VFIO based driver for Mediated device"
+static int vfio_mdev_notifier(struct notifier_block *nb, unsigned long action,
+ void *data)
+{
+ struct mdev_device *mdev = container_of(nb, struct mdev_device, nb);
+ struct parent_device *parent = mdev->parent;
+
+ return parent->ops->notifier(mdev, action, data);
+}
+
static int vfio_mdev_open(void *device_data)
{
struct mdev_device *mdev = device_data;
@@ -40,6 +49,11 @@ static int vfio_mdev_open(void *device_data)
if (ret)
module_put(THIS_MODULE);
+ if (likely(parent->ops->notifier)) {
+ mdev->nb.notifier_call = vfio_mdev_notifier;
+ if (vfio_register_notifier(&mdev->dev, &mdev->nb))
+ pr_err("Failed to register notifier for mdev\n");
+ }
return ret;
}
@@ -48,6 +62,11 @@ static void vfio_mdev_release(void *device_data)
struct mdev_device *mdev = device_data;
struct parent_device *parent = mdev->parent;
+ if (likely(parent->ops->notifier)) {
+ if (vfio_unregister_notifier(&mdev->dev, &mdev->nb))
+ pr_err("Failed to unregister notifier for mdev\n");
+ }
+
if (likely(parent->ops->release))
parent->ops->release(mdev);
diff --git a/include/linux/mdev.h b/include/linux/mdev.h
index 4900cc472364..665afe0a4c31 100644
--- a/include/linux/mdev.h
+++ b/include/linux/mdev.h
@@ -37,6 +37,7 @@ struct mdev_device {
struct kref ref;
struct list_head next;
struct kobject *type_kobj;
+ struct notifier_block nb;
};
/**
@@ -85,6 +86,12 @@ struct mdev_device {
* @mmap: mmap callback
* @mdev: mediated device structure
* @vma: vma structure
+ * @notifer: Notifier callback, currently only for
+ * VFIO_IOMMU_NOTIFY_DMA_UNMAP action notified duing
+ * DMA_UNMAP call on mapped iova range.
+ * @mdev: mediated device structure
+ * @action: Action for which notifier is called
+ * @data: Data associated with the notifier
* Parent device that support mediated device should be registered with mdev
* module with parent_ops structure.
**/
@@ -106,6 +113,8 @@ struct parent_ops {
ssize_t (*ioctl)(struct mdev_device *mdev, unsigned int cmd,
unsigned long arg);
int (*mmap)(struct mdev_device *mdev, struct vm_area_struct *vma);
+ int (*notifier)(struct mdev_device *mdev, unsigned long action,
+ void *data);
};
/* interface for exporting mdev supported type attributes */
--
2.7.0
^ permalink raw reply related
* [Qemu-devel] [PATCH v12 15/22] vfio: Introduce vfio_set_irqs_validate_and_prepare()
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Vendor driver using mediated device framework would use same mechnism to
validate and prepare IRQs. Introducing this function to reduce code
replication in multiple drivers.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: Ie201f269dda0713ca18a07dc4852500bd8b48309
---
drivers/vfio/vfio.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
include/linux/vfio.h | 4 ++++
2 files changed, 52 insertions(+)
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 3648fd74aa30..0afb58e26873 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -1858,6 +1858,54 @@ int vfio_info_add_capability(struct vfio_info_cap *caps, int cap_type_id,
}
EXPORT_SYMBOL(vfio_info_add_capability);
+int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs,
+ int max_irq_type, size_t *data_size)
+{
+ unsigned long minsz;
+ size_t size;
+
+ minsz = offsetofend(struct vfio_irq_set, count);
+
+ if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) ||
+ (hdr->count >= (U32_MAX - hdr->start)) ||
+ (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
+ VFIO_IRQ_SET_ACTION_TYPE_MASK)))
+ return -EINVAL;
+
+ if (data_size)
+ *data_size = 0;
+
+ if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs)
+ return -EINVAL;
+
+ switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
+ case VFIO_IRQ_SET_DATA_NONE:
+ size = 0;
+ break;
+ case VFIO_IRQ_SET_DATA_BOOL:
+ size = sizeof(uint8_t);
+ break;
+ case VFIO_IRQ_SET_DATA_EVENTFD:
+ size = sizeof(int32_t);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (size) {
+ if (hdr->argsz - minsz < hdr->count * size)
+ return -EINVAL;
+
+ if (!data_size)
+ return -EINVAL;
+
+ *data_size = hdr->count * size;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
+
/*
* Pin a set of guest PFNs and return their associated host PFNs for local
* domain only.
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index b1aaf69cb470..d4ce14d34ea9 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -114,6 +114,10 @@ extern void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset);
extern int vfio_info_add_capability(struct vfio_info_cap *caps,
int cap_type_id, void *cap_type);
+extern int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr,
+ int num_irqs, int max_irq_type,
+ size_t *data_size);
+
struct pci_dev;
#ifdef CONFIG_EEH
extern void vfio_spapr_pci_eeh_open(struct pci_dev *pdev);
--
2.7.0
^ permalink raw reply related
* [Qemu-devel] [PATCH v12 14/22] vfio_pci: Update vfio_pci to use vfio_info_add_capability()
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Update msix_sparse_mmap_cap() to use vfio_info_add_capability()
Update region type capability to use vfio_info_add_capability()
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I52bb28c7875a6da5a79ddad1843e6088aff58a45
---
drivers/vfio/pci/vfio_pci.c | 49 ++++++++++++++++++---------------------------
1 file changed, 19 insertions(+), 30 deletions(-)
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 031bc08d000d..03b5434f4d5b 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -558,10 +558,9 @@ static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,
struct vfio_info_cap *caps)
{
- struct vfio_info_cap_header *header;
struct vfio_region_info_cap_sparse_mmap *sparse;
size_t end, size;
- int nr_areas = 2, i = 0;
+ int nr_areas = 2, i = 0, ret;
end = pci_resource_len(vdev->pdev, vdev->msix_bar);
@@ -572,13 +571,10 @@ static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,
size = sizeof(*sparse) + (nr_areas * sizeof(*sparse->areas));
- header = vfio_info_cap_add(caps, size,
- VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1);
- if (IS_ERR(header))
- return PTR_ERR(header);
+ sparse = kzalloc(size, GFP_KERNEL);
+ if (!sparse)
+ return -ENOMEM;
- sparse = container_of(header,
- struct vfio_region_info_cap_sparse_mmap, header);
sparse->nr_areas = nr_areas;
if (vdev->msix_offset & PAGE_MASK) {
@@ -594,26 +590,11 @@ static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,
i++;
}
- return 0;
-}
-
-static int region_type_cap(struct vfio_pci_device *vdev,
- struct vfio_info_cap *caps,
- unsigned int type, unsigned int subtype)
-{
- struct vfio_info_cap_header *header;
- struct vfio_region_info_cap_type *cap;
-
- header = vfio_info_cap_add(caps, sizeof(*cap),
- VFIO_REGION_INFO_CAP_TYPE, 1);
- if (IS_ERR(header))
- return PTR_ERR(header);
-
- cap = container_of(header, struct vfio_region_info_cap_type, header);
- cap->type = type;
- cap->subtype = subtype;
+ ret = vfio_info_add_capability(caps, VFIO_REGION_INFO_CAP_SPARSE_MMAP,
+ sparse);
+ kfree(sparse);
- return 0;
+ return ret;
}
int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
@@ -752,6 +733,9 @@ static long vfio_pci_ioctl(void *device_data,
break;
default:
+ {
+ struct vfio_region_info_cap_type cap_type;
+
if (info.index >=
VFIO_PCI_NUM_REGIONS + vdev->num_regions)
return -EINVAL;
@@ -762,11 +746,16 @@ static long vfio_pci_ioctl(void *device_data,
info.size = vdev->region[i].size;
info.flags = vdev->region[i].flags;
- ret = region_type_cap(vdev, &caps,
- vdev->region[i].type,
- vdev->region[i].subtype);
+ cap_type.type = vdev->region[i].type;
+ cap_type.subtype = vdev->region[i].subtype;
+
+ ret = vfio_info_add_capability(&caps,
+ VFIO_REGION_INFO_CAP_TYPE,
+ &cap_type);
if (ret)
return ret;
+
+ }
}
if (caps.size) {
--
2.7.0
^ permalink raw reply related
* [PATCH v12 16/22] vfio_pci: Updated to use vfio_set_irqs_validate_and_prepare()
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Updated vfio_pci.c file to use vfio_set_irqs_validate_and_prepare()
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I9f3daba89d8dba5cb5b01a8cff420412f30686c7
---
drivers/vfio/pci/vfio_pci.c | 34 +++++++---------------------------
1 file changed, 7 insertions(+), 27 deletions(-)
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 03b5434f4d5b..dcd7c2a99618 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -818,45 +818,25 @@ static long vfio_pci_ioctl(void *device_data,
} else if (cmd == VFIO_DEVICE_SET_IRQS) {
struct vfio_irq_set hdr;
- size_t size;
u8 *data = NULL;
int max, ret = 0;
+ size_t data_size = 0;
minsz = offsetofend(struct vfio_irq_set, count);
if (copy_from_user(&hdr, (void __user *)arg, minsz))
return -EFAULT;
- if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS ||
- hdr.count >= (U32_MAX - hdr.start) ||
- hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
- VFIO_IRQ_SET_ACTION_TYPE_MASK))
- return -EINVAL;
-
max = vfio_pci_get_irq_count(vdev, hdr.index);
- if (hdr.start >= max || hdr.start + hdr.count > max)
- return -EINVAL;
- switch (hdr.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
- case VFIO_IRQ_SET_DATA_NONE:
- size = 0;
- break;
- case VFIO_IRQ_SET_DATA_BOOL:
- size = sizeof(uint8_t);
- break;
- case VFIO_IRQ_SET_DATA_EVENTFD:
- size = sizeof(int32_t);
- break;
- default:
- return -EINVAL;
- }
-
- if (size) {
- if (hdr.argsz - minsz < hdr.count * size)
- return -EINVAL;
+ ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+ VFIO_PCI_NUM_IRQS, &data_size);
+ if (ret)
+ return ret;
+ if (data_size) {
data = memdup_user((void __user *)(arg + minsz),
- hdr.count * size);
+ data_size);
if (IS_ERR(data))
return PTR_ERR(data);
}
--
2.7.0
^ permalink raw reply related
* [PATCH v12 10/22] vfio iommu type1: Add support for mediated devices
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
VFIO IOMMU drivers are designed for the devices which are IOMMU capable.
Mediated device only uses IOMMU APIs, the underlying hardware can be
managed by an IOMMU domain.
Aim of this change is:
- To use most of the code of TYPE1 IOMMU driver for mediated devices
- To support direct assigned device and mediated device in single module
This change adds pin and unpin support for mediated device to TYPE1 IOMMU
backend module. More details:
- Domain for external user is tracked separately in vfio_iommu structure.
It is allocated when group for first mdev device is attached.
- Pages pinned for external domain are tracked in each vfio_dma structure
for that iova range.
- Page tracking rb-tree in vfio_dma keeps <iova, pfn, ref_count>. Key of
rb-tree is iova, but it actually aims to track pfns.
- On external pin request for an iova, page is pinned only once, if iova
is already pinned and tracked, ref_count is incremented.
- External unin request unpins pages only when ref_count is 0.
- Pinned pages list is used to verify unpinning request and to unpin
remaining pages while detaching the group for that device.
- Page accounting is updated to account in its address space where the
pages are pinned/unpinned, i.e dma->task
- Accouting for mdev device is only done if there is no iommu capable
domain in the container. When there is a direct device assigned to the
container and that domain is iommu capable, all pages are already pinned
during DMA_MAP.
- Page accouting is updated on hot plug and unplug mdev device and pass
through device.
Tested by assigning below combinations of devices to a single VM:
- GPU pass through only
- vGPU device only
- One GPU pass through and one vGPU device
- Linux VM hot plug and unplug vGPU device while GPU pass through device
exist
- Linux VM hot plug and unplug GPU pass through device while vGPU device
exist
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I295d6f0f2e0579b8d9882bfd8fd5a4194b97bd9a
---
drivers/vfio/vfio_iommu_type1.c | 587 +++++++++++++++++++++++++++++++++++-----
1 file changed, 526 insertions(+), 61 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 50aca95cf61e..2697d874dd35 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -37,6 +37,7 @@
#include <linux/vfio.h>
#include <linux/workqueue.h>
#include <linux/pid_namespace.h>
+#include <linux/mdev.h>
#define DRIVER_VERSION "0.2"
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
@@ -56,6 +57,7 @@ MODULE_PARM_DESC(disable_hugepages,
struct vfio_iommu {
struct list_head domain_list;
+ struct vfio_domain *external_domain; /* domain for external user */
struct mutex lock;
struct rb_root dma_list;
bool v2;
@@ -76,7 +78,9 @@ struct vfio_dma {
unsigned long vaddr; /* Process virtual addr */
size_t size; /* Map size (bytes) */
int prot; /* IOMMU_READ/WRITE */
+ bool iommu_mapped;
struct task_struct *task;
+ struct rb_root pfn_list; /* Ex-user pinned pfn list */
};
struct vfio_group {
@@ -85,6 +89,21 @@ struct vfio_group {
};
/*
+ * Guest RAM pinning working set or DMA target
+ */
+struct vfio_pfn {
+ struct rb_node node;
+ dma_addr_t iova; /* Device address */
+ unsigned long pfn; /* Host pfn */
+ atomic_t ref_count;
+};
+
+#define IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu) \
+ (!list_empty(&iommu->domain_list))
+
+static int put_pfn(unsigned long pfn, int prot);
+
+/*
* This code handles mapping and unmapping of user data buffers
* into DMA'ble space using the IOMMU
*/
@@ -132,6 +151,97 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
rb_erase(&old->node, &iommu->dma_list);
}
+/*
+ * Helper Functions for host iova-pfn list
+ */
+static struct vfio_pfn *vfio_find_vpfn(struct vfio_dma *dma, dma_addr_t iova)
+{
+ struct vfio_pfn *vpfn;
+ struct rb_node *node = dma->pfn_list.rb_node;
+
+ while (node) {
+ vpfn = rb_entry(node, struct vfio_pfn, node);
+
+ if (iova < vpfn->iova)
+ node = node->rb_left;
+ else if (iova > vpfn->iova)
+ node = node->rb_right;
+ else
+ return vpfn;
+ }
+ return NULL;
+}
+
+static void vfio_link_pfn(struct vfio_dma *dma,
+ struct vfio_pfn *new)
+{
+ struct rb_node **link, *parent = NULL;
+ struct vfio_pfn *vpfn;
+
+ link = &dma->pfn_list.rb_node;
+ while (*link) {
+ parent = *link;
+ vpfn = rb_entry(parent, struct vfio_pfn, node);
+
+ if (new->iova < vpfn->iova)
+ link = &(*link)->rb_left;
+ else
+ link = &(*link)->rb_right;
+ }
+
+ rb_link_node(&new->node, parent, link);
+ rb_insert_color(&new->node, &dma->pfn_list);
+}
+
+static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old)
+{
+ rb_erase(&old->node, &dma->pfn_list);
+}
+
+static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova,
+ unsigned long pfn)
+{
+ struct vfio_pfn *vpfn;
+
+ vpfn = kzalloc(sizeof(*vpfn), GFP_KERNEL);
+ if (!vpfn)
+ return -ENOMEM;
+
+ vpfn->iova = iova;
+ vpfn->pfn = pfn;
+ atomic_set(&vpfn->ref_count, 1);
+ vfio_link_pfn(dma, vpfn);
+ return 0;
+}
+
+static void vfio_remove_from_pfn_list(struct vfio_dma *dma,
+ struct vfio_pfn *vpfn)
+{
+ vfio_unlink_pfn(dma, vpfn);
+ kfree(vpfn);
+}
+
+static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma,
+ unsigned long iova)
+{
+ struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
+
+ if (vpfn)
+ atomic_inc(&vpfn->ref_count);
+ return vpfn;
+}
+
+static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn)
+{
+ int ret = 0;
+
+ if (atomic_dec_and_test(&vpfn->ref_count)) {
+ ret = put_pfn(vpfn->pfn, dma->prot);
+ vfio_remove_from_pfn_list(dma, vpfn);
+ }
+ return ret;
+}
+
struct vwork {
struct mm_struct *mm;
long npage;
@@ -270,7 +380,6 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
}
up_read(&mm->mmap_sem);
-
return ret;
}
@@ -280,28 +389,35 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
* first page and all consecutive pages with the same locking.
*/
static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
- long npage, int prot, unsigned long *pfn_base)
+ long npage, unsigned long *pfn_base)
{
unsigned long limit;
bool lock_cap = ns_capable(task_active_pid_ns(dma->task)->user_ns,
CAP_IPC_LOCK);
struct mm_struct *mm;
- long ret, i;
+ long ret, i, lock_acct = 0;
bool rsvd;
+ struct vfio_pfn *vpfn;
+ dma_addr_t iova;
mm = get_task_mm(dma->task);
if (!mm)
return -ENODEV;
- ret = vaddr_get_pfn(mm, vaddr, prot, pfn_base);
+ ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);
if (ret)
goto pin_pg_remote_exit;
+ iova = vaddr - dma->vaddr + dma->iova;
+ vpfn = vfio_find_vpfn(dma, iova);
+ if (!vpfn)
+ lock_acct = 1;
+
rsvd = is_invalid_reserved_pfn(*pfn_base);
limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
- if (!rsvd && !lock_cap && mm->locked_vm + 1 > limit) {
- put_pfn(*pfn_base, prot);
+ if (!rsvd && !lock_cap && mm->locked_vm + lock_acct > limit) {
+ put_pfn(*pfn_base, dma->prot);
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
limit << PAGE_SHIFT);
ret = -ENOMEM;
@@ -310,7 +426,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
if (unlikely(disable_hugepages)) {
if (!rsvd)
- vfio_lock_acct(dma->task, 1);
+ vfio_lock_acct(dma->task, lock_acct);
ret = 1;
goto pin_pg_remote_exit;
}
@@ -319,18 +435,24 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
unsigned long pfn = 0;
- ret = vaddr_get_pfn(mm, vaddr, prot, &pfn);
+ ret = vaddr_get_pfn(mm, vaddr, dma->prot, &pfn);
if (ret)
break;
if (pfn != *pfn_base + i ||
rsvd != is_invalid_reserved_pfn(pfn)) {
- put_pfn(pfn, prot);
+ put_pfn(pfn, dma->prot);
break;
}
- if (!rsvd && !lock_cap && mm->locked_vm + i + 1 > limit) {
- put_pfn(pfn, prot);
+ iova = vaddr - dma->vaddr + dma->iova;
+ vpfn = vfio_find_vpfn(dma, iova);
+ if (!vpfn)
+ lock_acct++;
+
+ if (!rsvd && !lock_cap &&
+ mm->locked_vm + lock_acct + 1 > limit) {
+ put_pfn(pfn, dma->prot);
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
__func__, limit << PAGE_SHIFT);
break;
@@ -338,7 +460,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
}
if (!rsvd)
- vfio_lock_acct(dma->task, i);
+ vfio_lock_acct(dma->task, lock_acct);
ret = i;
pin_pg_remote_exit:
@@ -346,14 +468,78 @@ pin_pg_remote_exit:
return ret;
}
-static long vfio_unpin_pages_remote(struct vfio_dma *dma, unsigned long pfn,
- long npage, int prot, bool do_accounting)
+static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
+ unsigned long pfn, long npage,
+ bool do_accounting)
{
- unsigned long unlocked = 0;
+ long unlocked = 0, locked = 0;
long i;
- for (i = 0; i < npage; i++)
- unlocked += put_pfn(pfn++, prot);
+ for (i = 0; i < npage; i++) {
+ struct vfio_pfn *vpfn;
+
+ unlocked += put_pfn(pfn++, dma->prot);
+
+ vpfn = vfio_find_vpfn(dma, iova + (i << PAGE_SHIFT));
+ if (vpfn)
+ locked++;
+ }
+
+ if (do_accounting)
+ vfio_lock_acct(dma->task, locked - unlocked);
+
+ return unlocked;
+}
+
+static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
+ unsigned long *pfn_base, bool do_accounting)
+{
+ unsigned long limit;
+ bool lock_cap = ns_capable(task_active_pid_ns(dma->task)->user_ns,
+ CAP_IPC_LOCK);
+ struct mm_struct *mm;
+ int ret;
+ bool rsvd;
+
+ mm = get_task_mm(dma->task);
+ if (!mm)
+ return -ENODEV;
+
+ ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);
+ if (ret)
+ goto pin_page_exit;
+
+ rsvd = is_invalid_reserved_pfn(*pfn_base);
+ limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+
+ if (!rsvd && !lock_cap && mm->locked_vm + 1 > limit) {
+ put_pfn(*pfn_base, dma->prot);
+ pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK (%ld) exceeded\n",
+ __func__, dma->task->comm, task_pid_nr(dma->task),
+ limit << PAGE_SHIFT);
+ ret = -ENOMEM;
+ goto pin_page_exit;
+ }
+
+ if (!rsvd && do_accounting)
+ vfio_lock_acct(dma->task, 1);
+ ret = 1;
+
+pin_page_exit:
+ mmput(mm);
+ return ret;
+}
+
+static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova,
+ bool do_accounting)
+{
+ int unlocked;
+ struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
+
+ if (!vpfn)
+ return 0;
+
+ unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);
if (do_accounting)
vfio_lock_acct(dma->task, -unlocked);
@@ -361,14 +547,148 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, unsigned long pfn,
return unlocked;
}
-static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
+static int vfio_iommu_type1_pin_pages(void *iommu_data,
+ unsigned long *user_pfn,
+ int npage, int prot,
+ unsigned long *phys_pfn)
+{
+ struct vfio_iommu *iommu = iommu_data;
+ int i, j, ret;
+ unsigned long remote_vaddr;
+ unsigned long *pfn = phys_pfn;
+ struct vfio_dma *dma;
+ bool do_accounting;
+
+ if (!iommu || !user_pfn || !phys_pfn)
+ return -EINVAL;
+
+ /* Supported for v2 version only */
+ if (!iommu->v2)
+ return -EACCES;
+
+ mutex_lock(&iommu->lock);
+
+ if (!iommu->external_domain) {
+ ret = -EINVAL;
+ goto pin_done;
+ }
+
+ /*
+ * If iommu capable domain exist in the container then all pages are
+ * already pinned and accounted. Accouting should be done if there is no
+ * iommu capable domain in the container.
+ */
+ do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
+
+ for (i = 0; i < npage; i++) {
+ dma_addr_t iova;
+ struct vfio_pfn *vpfn;
+
+ iova = user_pfn[i] << PAGE_SHIFT;
+ dma = vfio_find_dma(iommu, iova, 0);
+ if (!dma) {
+ ret = -EINVAL;
+ goto pin_unwind;
+ }
+
+ if ((dma->prot & prot) != prot) {
+ pr_info("%s: dma->prot: 0x%x prot: 0x%x\n",
+ __func__, dma->prot, prot);
+ ret = -EPERM;
+ goto pin_unwind;
+ }
+
+ vpfn = vfio_iova_get_vfio_pfn(dma, iova);
+ if (vpfn) {
+ pfn[i] = vpfn->pfn;
+ continue;
+ }
+
+ remote_vaddr = dma->vaddr + iova - dma->iova;
+ ret = vfio_pin_page_external(dma, remote_vaddr, &pfn[i],
+ do_accounting);
+ if (ret <= 0) {
+ WARN_ON(!ret);
+ goto pin_unwind;
+ }
+
+ ret = vfio_add_to_pfn_list(dma, iova, pfn[i]);
+ if (ret) {
+ vfio_unpin_page_external(dma, iova, do_accounting);
+ goto pin_unwind;
+ }
+ }
+
+ ret = i;
+ goto pin_done;
+
+pin_unwind:
+ pfn[i] = 0;
+ for (j = 0; j < i; j++) {
+ dma_addr_t iova;
+
+ iova = user_pfn[j] << PAGE_SHIFT;
+ dma = vfio_find_dma(iommu, iova, 0);
+ vfio_unpin_page_external(dma, iova, do_accounting);
+ pfn[j] = 0;
+ }
+pin_done:
+ mutex_unlock(&iommu->lock);
+ return ret;
+}
+
+static int vfio_iommu_type1_unpin_pages(void *iommu_data,
+ unsigned long *user_pfn,
+ int npage)
+{
+ struct vfio_iommu *iommu = iommu_data;
+ bool do_accounting;
+ int unlocked = 0, i;
+
+ if (!iommu || !user_pfn)
+ return -EINVAL;
+
+ /* Supported for v2 version only */
+ if (!iommu->v2)
+ return -EACCES;
+
+ mutex_lock(&iommu->lock);
+
+ if (!iommu->external_domain) {
+ mutex_unlock(&iommu->lock);
+ return -EINVAL;
+ }
+
+ do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
+ for (i = 0; i < npage; i++) {
+ struct vfio_dma *dma;
+ dma_addr_t iova;
+
+ iova = user_pfn[i] << PAGE_SHIFT;
+ dma = vfio_find_dma(iommu, iova, 0);
+ if (!dma)
+ goto unpin_exit;
+ unlocked += vfio_unpin_page_external(dma, iova, do_accounting);
+ }
+
+unpin_exit:
+ mutex_unlock(&iommu->lock);
+ return unlocked;
+}
+
+static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
+ bool do_accounting)
{
dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
struct vfio_domain *domain, *d;
long unlocked = 0;
if (!dma->size)
- return;
+ return 0;
+
+ if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
+ return 0;
+
/*
* We use the IOMMU to track the physical addresses, otherwise we'd
* need a much more complicated tracking system. Unfortunately that
@@ -410,20 +730,26 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
if (WARN_ON(!unmapped))
break;
- unlocked += vfio_unpin_pages_remote(dma, phys >> PAGE_SHIFT,
+ unlocked += vfio_unpin_pages_remote(dma, iova,
+ phys >> PAGE_SHIFT,
unmapped >> PAGE_SHIFT,
- dma->prot, false);
+ false);
iova += unmapped;
cond_resched();
}
- vfio_lock_acct(dma->task, -unlocked);
+ dma->iommu_mapped = false;
+ if (do_accounting) {
+ vfio_lock_acct(dma->task, -unlocked);
+ return 0;
+ }
+ return unlocked;
}
static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
{
- vfio_unmap_unpin(iommu, dma);
+ vfio_unmap_unpin(iommu, dma, true);
vfio_unlink_dma(iommu, dma);
put_task_struct(dma->task);
kfree(dma);
@@ -606,8 +932,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
while (size) {
/* Pin a contiguous chunk of memory */
npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
- size >> PAGE_SHIFT, dma->prot,
- &pfn);
+ size >> PAGE_SHIFT, &pfn);
if (npage <= 0) {
WARN_ON(!npage);
ret = (int)npage;
@@ -618,8 +943,8 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,
dma->prot);
if (ret) {
- vfio_unpin_pages_remote(dma, pfn, npage,
- dma->prot, true);
+ vfio_unpin_pages_remote(dma, iova + dma->size, pfn,
+ npage, true);
break;
}
@@ -627,6 +952,8 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
dma->size += npage << PAGE_SHIFT;
}
+ dma->iommu_mapped = true;
+
if (ret)
vfio_remove_dma(iommu, dma);
@@ -682,11 +1009,16 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
dma->prot = prot;
get_task_struct(current);
dma->task = current;
+ dma->pfn_list = RB_ROOT;
/* Insert zero-sized and grow as we map chunks of it */
vfio_link_dma(iommu, dma);
- ret = vfio_pin_map_dma(iommu, dma, size);
+ /* Don't pin and map if container doesn't contain IOMMU capable domain*/
+ if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
+ dma->size = size;
+ else
+ ret = vfio_pin_map_dma(iommu, dma, size);
do_map_err:
mutex_unlock(&iommu->lock);
return ret;
@@ -715,10 +1047,6 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
d = list_first_entry(&iommu->domain_list, struct vfio_domain, next);
n = rb_first(&iommu->dma_list);
- /* If there's not a domain, there better not be any mappings */
- if (WARN_ON(n && !d))
- return -EINVAL;
-
for (; n; n = rb_next(n)) {
struct vfio_dma *dma;
dma_addr_t iova;
@@ -727,21 +1055,49 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
iova = dma->iova;
while (iova < dma->iova + dma->size) {
- phys_addr_t phys = iommu_iova_to_phys(d->domain, iova);
+ phys_addr_t phys;
size_t size;
- if (WARN_ON(!phys)) {
- iova += PAGE_SIZE;
- continue;
+ if (dma->iommu_mapped) {
+ phys_addr_t p;
+ dma_addr_t i;
+
+ phys = iommu_iova_to_phys(d->domain, iova);
+
+ if (WARN_ON(!phys)) {
+ iova += PAGE_SIZE;
+ continue;
+ }
+
+ size = PAGE_SIZE;
+ p = phys + size;
+ i = iova + size;
+ while (i < dma->iova + dma->size &&
+ p == iommu_iova_to_phys(d->domain, i)) {
+ size += PAGE_SIZE;
+ p += PAGE_SIZE;
+ i += PAGE_SIZE;
+ }
+ } else {
+ unsigned long pfn;
+ unsigned long vaddr = dma->vaddr +
+ (iova - dma->iova);
+ size_t n = dma->iova + dma->size - iova;
+ long npage;
+
+ npage = vfio_pin_pages_remote(dma, vaddr,
+ n >> PAGE_SHIFT,
+ &pfn);
+ if (npage <= 0) {
+ WARN_ON(!npage);
+ ret = (int)npage;
+ return ret;
+ }
+
+ phys = pfn << PAGE_SHIFT;
+ size = npage << PAGE_SHIFT;
}
- size = PAGE_SIZE;
-
- while (iova + size < dma->iova + dma->size &&
- phys + size == iommu_iova_to_phys(d->domain,
- iova + size))
- size += PAGE_SIZE;
-
ret = iommu_map(domain->domain, iova, phys,
size, dma->prot | domain->prot);
if (ret)
@@ -749,8 +1105,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
iova += size;
}
+ dma->iommu_mapped = true;
}
-
return 0;
}
@@ -806,7 +1162,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
struct vfio_iommu *iommu = iommu_data;
struct vfio_group *group;
struct vfio_domain *domain, *d;
- struct bus_type *bus = NULL;
+ struct bus_type *bus = NULL, *mdev_bus;
int ret;
mutex_lock(&iommu->lock);
@@ -818,6 +1174,13 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
}
}
+ if (iommu->external_domain) {
+ if (find_iommu_group(iommu->external_domain, iommu_group)) {
+ mutex_unlock(&iommu->lock);
+ return -EINVAL;
+ }
+ }
+
group = kzalloc(sizeof(*group), GFP_KERNEL);
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!group || !domain) {
@@ -832,6 +1195,25 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
if (ret)
goto out_free;
+ mdev_bus = symbol_get(mdev_bus_type);
+
+ if (mdev_bus) {
+ if ((bus == mdev_bus) && !iommu_present(bus)) {
+ symbol_put(mdev_bus_type);
+ if (!iommu->external_domain) {
+ INIT_LIST_HEAD(&domain->group_list);
+ iommu->external_domain = domain;
+ } else
+ kfree(domain);
+
+ list_add(&group->next,
+ &iommu->external_domain->group_list);
+ mutex_unlock(&iommu->lock);
+ return 0;
+ }
+ symbol_put(mdev_bus_type);
+ }
+
domain->domain = iommu_domain_alloc(bus);
if (!domain->domain) {
ret = -EIO;
@@ -922,6 +1304,46 @@ static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu)
vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node));
}
+static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu)
+{
+ struct rb_node *n, *p;
+
+ n = rb_first(&iommu->dma_list);
+ for (; n; n = rb_next(n)) {
+ struct vfio_dma *dma;
+ long locked = 0, unlocked = 0;
+
+ dma = rb_entry(n, struct vfio_dma, node);
+ unlocked += vfio_unmap_unpin(iommu, dma, false);
+ p = rb_first(&dma->pfn_list);
+ for (; p; p = rb_next(p))
+ locked++;
+ vfio_lock_acct(dma->task, locked - unlocked);
+ }
+}
+
+static void vfio_external_unpin_all(struct vfio_iommu *iommu,
+ bool do_accounting)
+{
+ struct rb_node *n, *p;
+
+ n = rb_first(&iommu->dma_list);
+ for (; n; n = rb_next(n)) {
+ struct vfio_dma *dma;
+
+ dma = rb_entry(n, struct vfio_dma, node);
+ while ((p = rb_first(&dma->pfn_list))) {
+ int unlocked;
+ struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn,
+ node);
+
+ unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);
+ if (do_accounting)
+ vfio_lock_acct(dma->task, -unlocked);
+ }
+ }
+}
+
static void vfio_iommu_type1_detach_group(void *iommu_data,
struct iommu_group *iommu_group)
{
@@ -931,6 +1353,26 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
mutex_lock(&iommu->lock);
+ if (iommu->external_domain) {
+ domain = iommu->external_domain;
+ group = find_iommu_group(domain, iommu_group);
+ if (group) {
+ list_del(&group->next);
+ kfree(group);
+
+ if (list_empty(&domain->group_list)) {
+ if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) {
+ vfio_external_unpin_all(iommu, true);
+ vfio_iommu_unmap_unpin_all(iommu);
+ } else
+ vfio_external_unpin_all(iommu, false);
+ kfree(domain);
+ iommu->external_domain = NULL;
+ }
+ goto detach_group_done;
+ }
+ }
+
list_for_each_entry(domain, &iommu->domain_list, next) {
group = find_iommu_group(domain, iommu_group);
if (!group)
@@ -940,21 +1382,27 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
list_del(&group->next);
kfree(group);
/*
- * Group ownership provides privilege, if the group
- * list is empty, the domain goes away. If it's the
- * last domain, then all the mappings go away too.
+ * Group ownership provides privilege, if the group list is
+ * empty, the domain goes away. If it's the last domain with
+ * iommu and external domain doesn't exist, then all the
+ * mappings go away too. If it's the last domain with iommu and
+ * external domain exist, update accounting
*/
if (list_empty(&domain->group_list)) {
- if (list_is_singular(&iommu->domain_list))
- vfio_iommu_unmap_unpin_all(iommu);
+ if (list_is_singular(&iommu->domain_list)) {
+ if (!iommu->external_domain)
+ vfio_iommu_unmap_unpin_all(iommu);
+ else
+ vfio_iommu_unmap_unpin_reaccount(iommu);
+ }
iommu_domain_free(domain->domain);
list_del(&domain->next);
kfree(domain);
}
- goto done;
+ break;
}
-done:
+detach_group_done:
mutex_unlock(&iommu->lock);
}
@@ -986,27 +1434,42 @@ static void *vfio_iommu_type1_open(unsigned long arg)
return iommu;
}
+static void vfio_release_domain(struct vfio_domain *domain, bool external)
+{
+ struct vfio_group *group, *group_tmp;
+
+ list_for_each_entry_safe(group, group_tmp,
+ &domain->group_list, next) {
+ if (!external)
+ iommu_detach_group(domain->domain, group->iommu_group);
+ list_del(&group->next);
+ kfree(group);
+ }
+
+ if (!external)
+ iommu_domain_free(domain->domain);
+}
+
static void vfio_iommu_type1_release(void *iommu_data)
{
struct vfio_iommu *iommu = iommu_data;
struct vfio_domain *domain, *domain_tmp;
- struct vfio_group *group, *group_tmp;
+
+ if (iommu->external_domain) {
+ vfio_release_domain(iommu->external_domain, true);
+ vfio_external_unpin_all(iommu, false);
+ kfree(iommu->external_domain);
+ iommu->external_domain = NULL;
+ }
vfio_iommu_unmap_unpin_all(iommu);
list_for_each_entry_safe(domain, domain_tmp,
&iommu->domain_list, next) {
- list_for_each_entry_safe(group, group_tmp,
- &domain->group_list, next) {
- iommu_detach_group(domain->domain, group->iommu_group);
- list_del(&group->next);
- kfree(group);
- }
- iommu_domain_free(domain->domain);
+ vfio_release_domain(domain, false);
list_del(&domain->next);
kfree(domain);
}
-
kfree(iommu);
}
@@ -1110,6 +1573,8 @@ static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
.ioctl = vfio_iommu_type1_ioctl,
.attach_group = vfio_iommu_type1_attach_group,
.detach_group = vfio_iommu_type1_detach_group,
+ .pin_pages = vfio_iommu_type1_pin_pages,
+ .unpin_pages = vfio_iommu_type1_unpin_pages,
};
static int __init vfio_iommu_type1_init(void)
--
2.7.0
^ permalink raw reply related
* [PATCH v12 09/22] vfio iommu type1: Add task structure to vfio_dma
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Add task structure to vfio_dma structure.
During DMA_UNMAP, same task who mapped it or other task who shares same
address space is allowed to unmap, otherwise unmap fails.
QEMU maps few iova ranges initially, then fork threads and from the child
thread calls DMA_UNMAP on previously mapped iova. Since child shares same
address space, DMA_UNMAP is successful.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I7600f1bea6b384fd589fa72421ccf031bcfd9ac5
---
drivers/vfio/vfio_iommu_type1.c | 137 +++++++++++++++++++++++++---------------
1 file changed, 86 insertions(+), 51 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index ffe2026f1341..50aca95cf61e 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -36,6 +36,7 @@
#include <linux/uaccess.h>
#include <linux/vfio.h>
#include <linux/workqueue.h>
+#include <linux/pid_namespace.h>
#define DRIVER_VERSION "0.2"
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
@@ -75,6 +76,7 @@ struct vfio_dma {
unsigned long vaddr; /* Process virtual addr */
size_t size; /* Map size (bytes) */
int prot; /* IOMMU_READ/WRITE */
+ struct task_struct *task;
};
struct vfio_group {
@@ -277,41 +279,47 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
* the iommu can only map chunks of consecutive pfns anyway, so get the
* first page and all consecutive pages with the same locking.
*/
-static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
- int prot, unsigned long *pfn_base)
+static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
+ long npage, int prot, unsigned long *pfn_base)
{
- unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
- bool lock_cap = capable(CAP_IPC_LOCK);
+ unsigned long limit;
+ bool lock_cap = ns_capable(task_active_pid_ns(dma->task)->user_ns,
+ CAP_IPC_LOCK);
+ struct mm_struct *mm;
long ret, i;
bool rsvd;
- if (!current->mm)
+ mm = get_task_mm(dma->task);
+ if (!mm)
return -ENODEV;
- ret = vaddr_get_pfn(current->mm, vaddr, prot, pfn_base);
+ ret = vaddr_get_pfn(mm, vaddr, prot, pfn_base);
if (ret)
- return ret;
+ goto pin_pg_remote_exit;
rsvd = is_invalid_reserved_pfn(*pfn_base);
+ limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
- if (!rsvd && !lock_cap && current->mm->locked_vm + 1 > limit) {
+ if (!rsvd && !lock_cap && mm->locked_vm + 1 > limit) {
put_pfn(*pfn_base, prot);
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
limit << PAGE_SHIFT);
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto pin_pg_remote_exit;
}
if (unlikely(disable_hugepages)) {
if (!rsvd)
- vfio_lock_acct(current, 1);
- return 1;
+ vfio_lock_acct(dma->task, 1);
+ ret = 1;
+ goto pin_pg_remote_exit;
}
/* Lock all the consecutive pages from pfn_base */
for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
unsigned long pfn = 0;
- ret = vaddr_get_pfn(current->mm, vaddr, prot, &pfn);
+ ret = vaddr_get_pfn(mm, vaddr, prot, &pfn);
if (ret)
break;
@@ -321,8 +329,7 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
break;
}
- if (!rsvd && !lock_cap &&
- current->mm->locked_vm + i + 1 > limit) {
+ if (!rsvd && !lock_cap && mm->locked_vm + i + 1 > limit) {
put_pfn(pfn, prot);
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
__func__, limit << PAGE_SHIFT);
@@ -331,13 +338,16 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
}
if (!rsvd)
- vfio_lock_acct(current, i);
+ vfio_lock_acct(dma->task, i);
+ ret = i;
- return i;
+pin_pg_remote_exit:
+ mmput(mm);
+ return ret;
}
-static long vfio_unpin_pages_remote(unsigned long pfn, long npage,
- int prot, bool do_accounting)
+static long vfio_unpin_pages_remote(struct vfio_dma *dma, unsigned long pfn,
+ long npage, int prot, bool do_accounting)
{
unsigned long unlocked = 0;
long i;
@@ -346,7 +356,7 @@ static long vfio_unpin_pages_remote(unsigned long pfn, long npage,
unlocked += put_pfn(pfn++, prot);
if (do_accounting)
- vfio_lock_acct(current, -unlocked);
+ vfio_lock_acct(dma->task, -unlocked);
return unlocked;
}
@@ -400,7 +410,7 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
if (WARN_ON(!unmapped))
break;
- unlocked += vfio_unpin_pages_remote(phys >> PAGE_SHIFT,
+ unlocked += vfio_unpin_pages_remote(dma, phys >> PAGE_SHIFT,
unmapped >> PAGE_SHIFT,
dma->prot, false);
iova += unmapped;
@@ -408,13 +418,14 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
cond_resched();
}
- vfio_lock_acct(current, -unlocked);
+ vfio_lock_acct(dma->task, -unlocked);
}
static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
{
vfio_unmap_unpin(iommu, dma);
vfio_unlink_dma(iommu, dma);
+ put_task_struct(dma->task);
kfree(dma);
}
@@ -510,6 +521,12 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) {
if (!iommu->v2 && unmap->iova > dma->iova)
break;
+ /*
+ * Task with same address space who mapped this iova range is
+ * allowed to unmap the iova range.
+ */
+ if (dma->task->mm != current->mm)
+ break;
unmapped += dma->size;
vfio_remove_dma(iommu, dma);
}
@@ -576,17 +593,55 @@ unwind:
return ret;
}
+static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
+ size_t map_size)
+{
+ dma_addr_t iova = dma->iova;
+ unsigned long vaddr = dma->vaddr;
+ size_t size = map_size;
+ long npage;
+ unsigned long pfn;
+ int ret = 0;
+
+ while (size) {
+ /* Pin a contiguous chunk of memory */
+ npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
+ size >> PAGE_SHIFT, dma->prot,
+ &pfn);
+ if (npage <= 0) {
+ WARN_ON(!npage);
+ ret = (int)npage;
+ break;
+ }
+
+ /* Map it! */
+ ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,
+ dma->prot);
+ if (ret) {
+ vfio_unpin_pages_remote(dma, pfn, npage,
+ dma->prot, true);
+ break;
+ }
+
+ size -= npage << PAGE_SHIFT;
+ dma->size += npage << PAGE_SHIFT;
+ }
+
+ if (ret)
+ vfio_remove_dma(iommu, dma);
+
+ return ret;
+}
+
static int vfio_dma_do_map(struct vfio_iommu *iommu,
struct vfio_iommu_type1_dma_map *map)
{
dma_addr_t iova = map->iova;
unsigned long vaddr = map->vaddr;
size_t size = map->size;
- long npage;
int ret = 0, prot = 0;
uint64_t mask;
struct vfio_dma *dma;
- unsigned long pfn;
/* Verify that none of our __u64 fields overflow */
if (map->size != size || map->vaddr != vaddr || map->iova != iova)
@@ -612,47 +667,27 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
mutex_lock(&iommu->lock);
if (vfio_find_dma(iommu, iova, size)) {
- mutex_unlock(&iommu->lock);
- return -EEXIST;
+ ret = -EEXIST;
+ goto do_map_err;
}
dma = kzalloc(sizeof(*dma), GFP_KERNEL);
if (!dma) {
- mutex_unlock(&iommu->lock);
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto do_map_err;
}
dma->iova = iova;
dma->vaddr = vaddr;
dma->prot = prot;
+ get_task_struct(current);
+ dma->task = current;
/* Insert zero-sized and grow as we map chunks of it */
vfio_link_dma(iommu, dma);
- while (size) {
- /* Pin a contiguous chunk of memory */
- npage = vfio_pin_pages_remote(vaddr + dma->size,
- size >> PAGE_SHIFT, prot, &pfn);
- if (npage <= 0) {
- WARN_ON(!npage);
- ret = (int)npage;
- break;
- }
-
- /* Map it! */
- ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage, prot);
- if (ret) {
- vfio_unpin_pages_remote(pfn, npage, prot, true);
- break;
- }
-
- size -= npage << PAGE_SHIFT;
- dma->size += npage << PAGE_SHIFT;
- }
-
- if (ret)
- vfio_remove_dma(iommu, dma);
-
+ ret = vfio_pin_map_dma(iommu, dma, size);
+do_map_err:
mutex_unlock(&iommu->lock);
return ret;
}
--
2.7.0
^ permalink raw reply related
* [PATCH v12 08/22] vfio iommu type1: Add find_iommu_group() function
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Add find_iommu_group()
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Reviewed-by: Jike Song <jike.song@intel.com>
Reviewed-by: Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
Change-Id: I9d372f1ebe9eb01a5a21374b8a2b03f7df73601f
---
drivers/vfio/vfio_iommu_type1.c | 57 ++++++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 24 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 52af5fc01d91..ffe2026f1341 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -752,11 +752,24 @@ static void vfio_test_domain_fgsp(struct vfio_domain *domain)
__free_pages(pages, order);
}
+static struct vfio_group *find_iommu_group(struct vfio_domain *domain,
+ struct iommu_group *iommu_group)
+{
+ struct vfio_group *g;
+
+ list_for_each_entry(g, &domain->group_list, next) {
+ if (g->iommu_group == iommu_group)
+ return g;
+ }
+
+ return NULL;
+}
+
static int vfio_iommu_type1_attach_group(void *iommu_data,
struct iommu_group *iommu_group)
{
struct vfio_iommu *iommu = iommu_data;
- struct vfio_group *group, *g;
+ struct vfio_group *group;
struct vfio_domain *domain, *d;
struct bus_type *bus = NULL;
int ret;
@@ -764,10 +777,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
mutex_lock(&iommu->lock);
list_for_each_entry(d, &iommu->domain_list, next) {
- list_for_each_entry(g, &d->group_list, next) {
- if (g->iommu_group != iommu_group)
- continue;
-
+ if (find_iommu_group(d, iommu_group)) {
mutex_unlock(&iommu->lock);
return -EINVAL;
}
@@ -887,27 +897,26 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
mutex_lock(&iommu->lock);
list_for_each_entry(domain, &iommu->domain_list, next) {
- list_for_each_entry(group, &domain->group_list, next) {
- if (group->iommu_group != iommu_group)
- continue;
+ group = find_iommu_group(domain, iommu_group);
+ if (!group)
+ continue;
- iommu_detach_group(domain->domain, iommu_group);
- list_del(&group->next);
- kfree(group);
- /*
- * Group ownership provides privilege, if the group
- * list is empty, the domain goes away. If it's the
- * last domain, then all the mappings go away too.
- */
- if (list_empty(&domain->group_list)) {
- if (list_is_singular(&iommu->domain_list))
- vfio_iommu_unmap_unpin_all(iommu);
- iommu_domain_free(domain->domain);
- list_del(&domain->next);
- kfree(domain);
- }
- goto done;
+ iommu_detach_group(domain->domain, iommu_group);
+ list_del(&group->next);
+ kfree(group);
+ /*
+ * Group ownership provides privilege, if the group
+ * list is empty, the domain goes away. If it's the
+ * last domain, then all the mappings go away too.
+ */
+ if (list_empty(&domain->group_list)) {
+ if (list_is_singular(&iommu->domain_list))
+ vfio_iommu_unmap_unpin_all(iommu);
+ iommu_domain_free(domain->domain);
+ list_del(&domain->next);
+ kfree(domain);
}
+ goto done;
}
done:
--
2.7.0
^ permalink raw reply related
* [PATCH v12 07/22] vfio iommu type1: Update argument of vaddr_get_pfn()
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Update arguments of vaddr_get_pfn() to take struct mm_struct *mm as input
argument.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I885fd4cd4a9f66f4ee2c1caf58267464ec239f52
---
drivers/vfio/vfio_iommu_type1.c | 32 ++++++++++++++++++++++++--------
1 file changed, 24 insertions(+), 8 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 34d17e51dc97..52af5fc01d91 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -230,20 +230,36 @@ static int put_pfn(unsigned long pfn, int prot)
return 0;
}
-static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
+static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
+ int prot, unsigned long *pfn)
{
struct page *page[1];
struct vm_area_struct *vma;
- int ret = -EFAULT;
+ int ret;
+
+ if (mm == current->mm) {
+ ret = get_user_pages_fast(vaddr, 1, !!(prot & IOMMU_WRITE),
+ page);
+ } else {
+ unsigned int flags = 0;
+
+ if (prot & IOMMU_WRITE)
+ flags |= FOLL_WRITE;
+
+ down_read(&mm->mmap_sem);
+ ret = get_user_pages_remote(NULL, mm, vaddr, 1, flags, page,
+ NULL);
+ up_read(&mm->mmap_sem);
+ }
- if (get_user_pages_fast(vaddr, 1, !!(prot & IOMMU_WRITE), page) == 1) {
+ if (ret == 1) {
*pfn = page_to_pfn(page[0]);
return 0;
}
- down_read(¤t->mm->mmap_sem);
+ down_read(&mm->mmap_sem);
- vma = find_vma_intersection(current->mm, vaddr, vaddr + 1);
+ vma = find_vma_intersection(mm, vaddr, vaddr + 1);
if (vma && vma->vm_flags & VM_PFNMAP) {
*pfn = ((vaddr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
@@ -251,7 +267,7 @@ static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
ret = 0;
}
- up_read(¤t->mm->mmap_sem);
+ up_read(&mm->mmap_sem);
return ret;
}
@@ -272,7 +288,7 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
if (!current->mm)
return -ENODEV;
- ret = vaddr_get_pfn(vaddr, prot, pfn_base);
+ ret = vaddr_get_pfn(current->mm, vaddr, prot, pfn_base);
if (ret)
return ret;
@@ -295,7 +311,7 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
unsigned long pfn = 0;
- ret = vaddr_get_pfn(vaddr, prot, &pfn);
+ ret = vaddr_get_pfn(current->mm, vaddr, prot, &pfn);
if (ret)
break;
--
2.7.0
^ permalink raw reply related
* [PATCH v12 06/22] vfio iommu type1: Update arguments of vfio_lock_acct
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: jike.song, kvm, linux-kernel, kevin.tian, qemu-devel,
Kirti Wankhede, bjsdjshi
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Added task structure as input argument to vfio_lock_acct() function.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I5d3673cc9d3786bb436b395d5f74537f1a36da80
---
drivers/vfio/vfio_iommu_type1.c | 30 ++++++++++++++++--------------
1 file changed, 16 insertions(+), 14 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 9f3d58d3dfaf..34d17e51dc97 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -150,17 +150,22 @@ static void vfio_lock_acct_bg(struct work_struct *work)
kfree(vwork);
}
-static void vfio_lock_acct(long npage)
+static void vfio_lock_acct(struct task_struct *task, long npage)
{
struct vwork *vwork;
struct mm_struct *mm;
- if (!current->mm || !npage)
+ if (!npage)
+ return;
+
+ mm = get_task_mm(task);
+ if (!mm)
return; /* process exited or nothing to do */
- if (down_write_trylock(¤t->mm->mmap_sem)) {
- current->mm->locked_vm += npage;
- up_write(¤t->mm->mmap_sem);
+ if (down_write_trylock(&mm->mmap_sem)) {
+ mm->locked_vm += npage;
+ up_write(&mm->mmap_sem);
+ mmput(mm);
return;
}
@@ -170,11 +175,8 @@ static void vfio_lock_acct(long npage)
* wouldn't need this silliness
*/
vwork = kmalloc(sizeof(struct vwork), GFP_KERNEL);
- if (!vwork)
- return;
- mm = get_task_mm(current);
- if (!mm) {
- kfree(vwork);
+ if (!vwork) {
+ mmput(mm);
return;
}
INIT_WORK(&vwork->work, vfio_lock_acct_bg);
@@ -285,7 +287,7 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
if (unlikely(disable_hugepages)) {
if (!rsvd)
- vfio_lock_acct(1);
+ vfio_lock_acct(current, 1);
return 1;
}
@@ -313,7 +315,7 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
}
if (!rsvd)
- vfio_lock_acct(i);
+ vfio_lock_acct(current, i);
return i;
}
@@ -328,7 +330,7 @@ static long vfio_unpin_pages_remote(unsigned long pfn, long npage,
unlocked += put_pfn(pfn++, prot);
if (do_accounting)
- vfio_lock_acct(-unlocked);
+ vfio_lock_acct(current, -unlocked);
return unlocked;
}
@@ -390,7 +392,7 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
cond_resched();
}
- vfio_lock_acct(-unlocked);
+ vfio_lock_acct(current, -unlocked);
}
static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
--
2.7.0
^ permalink raw reply related
* [PATCH v12 05/22] vfio iommu: Added pin and unpin callback functions to vfio_iommu_driver_ops
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Added APIs for pining and unpining set of pages. These call back into
backend iommu module to actually pin and unpin pages.
Added two new callback functions to struct vfio_iommu_driver_ops. Backend
IOMMU module that supports pining and unpinning pages for mdev devices
should provide these functions.
Renamed static functions in vfio_type1_iommu.c to resolve conflicts
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: Ia7417723aaae86bec2959ad9ae6c2915ddd340e0
---
drivers/vfio/vfio.c | 103 ++++++++++++++++++++++++++++++++++++++++
drivers/vfio/vfio_iommu_type1.c | 20 ++++----
include/linux/vfio.h | 12 ++++-
3 files changed, 124 insertions(+), 11 deletions(-)
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 2e83bdf007fe..7dcfbca2016a 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -1799,6 +1799,109 @@ void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
}
EXPORT_SYMBOL_GPL(vfio_info_cap_shift);
+
+/*
+ * Pin a set of guest PFNs and return their associated host PFNs for local
+ * domain only.
+ * @dev [in] : device
+ * @user_pfn [in]: array of user/guest PFNs to be unpinned. Number of user/guest
+ * PFNs should not be greater than PAGE_SIZE.
+ * @npage [in] :count of elements in array. This count should not be greater
+ * than PAGE_SIZE.
+ * @prot [in] : protection flags
+ * @phys_pfn[out] : array of host PFNs
+ * Return error or number of pages pinned.
+ */
+int vfio_pin_pages(struct device *dev, unsigned long *user_pfn, int npage,
+ int prot, unsigned long *phys_pfn)
+{
+ struct vfio_container *container;
+ struct vfio_group *group;
+ struct vfio_iommu_driver *driver;
+ int ret;
+
+ if (!dev || !user_pfn || !phys_pfn || !npage)
+ return -EINVAL;
+
+ if (npage >= PAGE_SIZE)
+ return -E2BIG;
+
+ group = vfio_group_get_from_dev(dev);
+ if (IS_ERR(group))
+ return PTR_ERR(group);
+
+ ret = vfio_group_add_container_user(group);
+ if (ret)
+ goto err_pin_pages;
+
+ container = group->container;
+ down_read(&container->group_lock);
+
+ driver = container->iommu_driver;
+ if (likely(driver && driver->ops->pin_pages))
+ ret = driver->ops->pin_pages(container->iommu_data, user_pfn,
+ npage, prot, phys_pfn);
+ else
+ ret = -ENOTTY;
+
+ up_read(&container->group_lock);
+ vfio_group_try_dissolve_container(group);
+
+err_pin_pages:
+ vfio_group_put(group);
+ return ret;
+}
+EXPORT_SYMBOL(vfio_pin_pages);
+
+/*
+ * Unpin set of host PFNs for local domain only.
+ * @dev [in] : device
+ * @user_pfn [in]: array of user/guest PFNs to be unpinned. Number of user/guest
+ * PFNs should not be greater than PAGE_SIZE.
+ * @npage [in] :count of elements in array. This count should not be greater
+ * than PAGE_SIZE.
+ * Return error or number of pages unpinned.
+ */
+int vfio_unpin_pages(struct device *dev, unsigned long *user_pfn, int npage)
+{
+ struct vfio_container *container;
+ struct vfio_group *group;
+ struct vfio_iommu_driver *driver;
+ int ret;
+
+ if (!dev || !user_pfn || !npage)
+ return -EINVAL;
+
+ if (npage >= PAGE_SIZE)
+ return -E2BIG;
+
+ group = vfio_group_get_from_dev(dev);
+ if (IS_ERR(group))
+ return PTR_ERR(group);
+
+ ret = vfio_group_add_container_user(group);
+ if (ret)
+ goto err_unpin_pages;
+
+ container = group->container;
+ down_read(&container->group_lock);
+
+ driver = container->iommu_driver;
+ if (likely(driver && driver->ops->unpin_pages))
+ ret = driver->ops->unpin_pages(container->iommu_data, user_pfn,
+ npage);
+ else
+ ret = -ENOTTY;
+
+ up_read(&container->group_lock);
+ vfio_group_try_dissolve_container(group);
+
+err_unpin_pages:
+ vfio_group_put(group);
+ return ret;
+}
+EXPORT_SYMBOL(vfio_unpin_pages);
+
/**
* Module/class support
*/
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 2ba19424e4a1..9f3d58d3dfaf 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -259,8 +259,8 @@ static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
* the iommu can only map chunks of consecutive pfns anyway, so get the
* first page and all consecutive pages with the same locking.
*/
-static long vfio_pin_pages(unsigned long vaddr, long npage,
- int prot, unsigned long *pfn_base)
+static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
+ int prot, unsigned long *pfn_base)
{
unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
bool lock_cap = capable(CAP_IPC_LOCK);
@@ -318,8 +318,8 @@ static long vfio_pin_pages(unsigned long vaddr, long npage,
return i;
}
-static long vfio_unpin_pages(unsigned long pfn, long npage,
- int prot, bool do_accounting)
+static long vfio_unpin_pages_remote(unsigned long pfn, long npage,
+ int prot, bool do_accounting)
{
unsigned long unlocked = 0;
long i;
@@ -382,9 +382,9 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
if (WARN_ON(!unmapped))
break;
- unlocked += vfio_unpin_pages(phys >> PAGE_SHIFT,
- unmapped >> PAGE_SHIFT,
- dma->prot, false);
+ unlocked += vfio_unpin_pages_remote(phys >> PAGE_SHIFT,
+ unmapped >> PAGE_SHIFT,
+ dma->prot, false);
iova += unmapped;
cond_resched();
@@ -613,8 +613,8 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
while (size) {
/* Pin a contiguous chunk of memory */
- npage = vfio_pin_pages(vaddr + dma->size,
- size >> PAGE_SHIFT, prot, &pfn);
+ npage = vfio_pin_pages_remote(vaddr + dma->size,
+ size >> PAGE_SHIFT, prot, &pfn);
if (npage <= 0) {
WARN_ON(!npage);
ret = (int)npage;
@@ -624,7 +624,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
/* Map it! */
ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage, prot);
if (ret) {
- vfio_unpin_pages(pfn, npage, prot, true);
+ vfio_unpin_pages_remote(pfn, npage, prot, true);
break;
}
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 0ecae0b1cd34..86f507d0f585 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -75,7 +75,11 @@ struct vfio_iommu_driver_ops {
struct iommu_group *group);
void (*detach_group)(void *iommu_data,
struct iommu_group *group);
-
+ int (*pin_pages)(void *iommu_data, unsigned long *user_pfn,
+ int npage, int prot,
+ unsigned long *phys_pfn);
+ int (*unpin_pages)(void *iommu_data,
+ unsigned long *user_pfn, int npage);
};
extern int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops);
@@ -127,6 +131,12 @@ static inline long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group,
}
#endif /* CONFIG_EEH */
+extern int vfio_pin_pages(struct device *dev, unsigned long *user_pfn,
+ int npage, int prot, unsigned long *phys_pfn);
+
+extern int vfio_unpin_pages(struct device *dev, unsigned long *user_pfn,
+ int npage);
+
/*
* IRQfd - generic
*/
--
2.7.0
^ permalink raw reply related
* [PATCH v12 04/22] vfio: Common function to increment container_users
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
This change rearrange functions to have common function to increment
container_users
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Reviewed-by: Jike Song <jike.song@intel.com>
Change-Id: I8bdeb352bc8439b107ffd519480fd4dc238677f2
---
drivers/vfio/vfio.c | 34 +++++++++++++++++++++-------------
1 file changed, 21 insertions(+), 13 deletions(-)
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 23bc86c1d05d..2e83bdf007fe 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -1385,6 +1385,23 @@ static bool vfio_group_viable(struct vfio_group *group)
group, vfio_dev_viable) == 0);
}
+static int vfio_group_add_container_user(struct vfio_group *group)
+{
+ if (!atomic_inc_not_zero(&group->container_users))
+ return -EINVAL;
+
+ if (group->noiommu) {
+ atomic_dec(&group->container_users);
+ return -EPERM;
+ }
+ if (!group->container->iommu_driver || !vfio_group_viable(group)) {
+ atomic_dec(&group->container_users);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static const struct file_operations vfio_device_fops;
static int vfio_group_get_device_fd(struct vfio_group *group, char *buf)
@@ -1694,23 +1711,14 @@ static const struct file_operations vfio_device_fops = {
struct vfio_group *vfio_group_get_external_user(struct file *filep)
{
struct vfio_group *group = filep->private_data;
+ int ret;
if (filep->f_op != &vfio_group_fops)
return ERR_PTR(-EINVAL);
- if (!atomic_inc_not_zero(&group->container_users))
- return ERR_PTR(-EINVAL);
-
- if (group->noiommu) {
- atomic_dec(&group->container_users);
- return ERR_PTR(-EPERM);
- }
-
- if (!group->container->iommu_driver ||
- !vfio_group_viable(group)) {
- atomic_dec(&group->container_users);
- return ERR_PTR(-EINVAL);
- }
+ ret = vfio_group_add_container_user(group);
+ if (ret)
+ return ERR_PTR(ret);
vfio_group_get(group);
--
2.7.0
^ permalink raw reply related
* [Qemu-devel] [PATCH v12 13/22] vfio: Introduce common function to add capabilities
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Vendor driver using mediated device framework should use
vfio_info_add_capability() to add capabilities.
Introduced this function to reduce code duplication in vendor drivers.
vfio_info_cap_shift() manipulated a data buffer to add an offset to each
element in a chain. This data buffer is documented in a uapi header.
Changing vfio_info_cap_shift symbol to be available to all drivers.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I6fca329fa2291f37a2c859d0bc97574d9e2ce1a6
---
drivers/vfio/vfio.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++-
include/linux/vfio.h | 3 +++
2 files changed, 62 insertions(+), 1 deletion(-)
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 388a3cbddbd9..3648fd74aa30 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -1797,8 +1797,66 @@ void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
for (tmp = caps->buf; tmp->next; tmp = (void *)tmp + tmp->next - offset)
tmp->next += offset;
}
-EXPORT_SYMBOL_GPL(vfio_info_cap_shift);
+EXPORT_SYMBOL(vfio_info_cap_shift);
+static int sparse_mmap_cap(struct vfio_info_cap *caps, void *cap_type)
+{
+ struct vfio_info_cap_header *header;
+ struct vfio_region_info_cap_sparse_mmap *sparse_cap, *sparse = cap_type;
+ size_t size;
+
+ size = sizeof(*sparse) + sparse->nr_areas * sizeof(*sparse->areas);
+ header = vfio_info_cap_add(caps, size,
+ VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1);
+ if (IS_ERR(header))
+ return PTR_ERR(header);
+
+ sparse_cap = container_of(header,
+ struct vfio_region_info_cap_sparse_mmap, header);
+ sparse_cap->nr_areas = sparse->nr_areas;
+ memcpy(sparse_cap->areas, sparse->areas,
+ sparse->nr_areas * sizeof(*sparse->areas));
+ return 0;
+}
+
+static int region_type_cap(struct vfio_info_cap *caps, void *cap_type)
+{
+ struct vfio_info_cap_header *header;
+ struct vfio_region_info_cap_type *type_cap, *cap = cap_type;
+
+ header = vfio_info_cap_add(caps, sizeof(*cap),
+ VFIO_REGION_INFO_CAP_TYPE, 1);
+ if (IS_ERR(header))
+ return PTR_ERR(header);
+
+ type_cap = container_of(header, struct vfio_region_info_cap_type,
+ header);
+ type_cap->type = cap->type;
+ type_cap->subtype = cap->subtype;
+ return 0;
+}
+
+int vfio_info_add_capability(struct vfio_info_cap *caps, int cap_type_id,
+ void *cap_type)
+{
+ int ret = -EINVAL;
+
+ if (!cap_type)
+ return 0;
+
+ switch (cap_type_id) {
+ case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
+ ret = sparse_mmap_cap(caps, cap_type);
+ break;
+
+ case VFIO_REGION_INFO_CAP_TYPE:
+ ret = region_type_cap(caps, cap_type);
+ break;
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL(vfio_info_add_capability);
/*
* Pin a set of guest PFNs and return their associated host PFNs for local
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 123e089388e9..b1aaf69cb470 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -111,6 +111,9 @@ extern struct vfio_info_cap_header *vfio_info_cap_add(
struct vfio_info_cap *caps, size_t size, u16 id, u16 version);
extern void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset);
+extern int vfio_info_add_capability(struct vfio_info_cap *caps,
+ int cap_type_id, void *cap_type);
+
struct pci_dev;
#ifdef CONFIG_EEH
extern void vfio_spapr_pci_eeh_open(struct pci_dev *pdev);
--
2.7.0
^ permalink raw reply related
* [PATCH v12 02/22] vfio: VFIO based driver for Mediated devices
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
vfio_mdev driver registers with mdev core driver.
mdev core driver creates mediated device and calls probe routine of
vfio_mdev driver for each device.
Probe routine of vfio_mdev driver adds mediated device to VFIO core module
This driver forms a shim layer that pass through VFIO devices operations
to vendor driver for mediated devices.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Reviewed-by: Jike Song <jike.song@intel.com>
Change-Id: I583f4734752971d3d112324d69e2508c88f359ec
---
drivers/vfio/mdev/Kconfig | 7 ++
drivers/vfio/mdev/Makefile | 1 +
drivers/vfio/mdev/mdev_core.c | 16 ++++-
drivers/vfio/mdev/vfio_mdev.c | 148 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 171 insertions(+), 1 deletion(-)
create mode 100644 drivers/vfio/mdev/vfio_mdev.c
diff --git a/drivers/vfio/mdev/Kconfig b/drivers/vfio/mdev/Kconfig
index 258481d65ebd..1aa0391d74f2 100644
--- a/drivers/vfio/mdev/Kconfig
+++ b/drivers/vfio/mdev/Kconfig
@@ -7,3 +7,10 @@ config VFIO_MDEV
Provides a framework to virtualize devices.
If you don't know what do here, say N.
+
+config VFIO_MDEV_DEVICE
+ tristate "VFIO support for Mediated devices"
+ depends on VFIO && VFIO_MDEV
+ default n
+ help
+ VFIO based driver for mediated devices.
diff --git a/drivers/vfio/mdev/Makefile b/drivers/vfio/mdev/Makefile
index 31bc04801d94..fa2d5ea466ee 100644
--- a/drivers/vfio/mdev/Makefile
+++ b/drivers/vfio/mdev/Makefile
@@ -2,3 +2,4 @@
mdev-y := mdev_core.o mdev_sysfs.o mdev_driver.o
obj-$(CONFIG_VFIO_MDEV) += mdev.o
+obj-$(CONFIG_VFIO_MDEV_DEVICE) += vfio_mdev.o
diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c
index 613e8a8a3b2a..1e0714ebc56a 100644
--- a/drivers/vfio/mdev/mdev_core.c
+++ b/drivers/vfio/mdev/mdev_core.c
@@ -354,7 +354,21 @@ int mdev_device_remove(struct device *dev, bool force_remove)
static int __init mdev_init(void)
{
- return mdev_bus_register();
+ int ret;
+
+ ret = mdev_bus_register();
+ if (ret) {
+ pr_err("Failed to register mdev bus\n");
+ return ret;
+ }
+
+ /*
+ * Attempt to load known vfio_mdev. This gives us a working environment
+ * without the user needing to explicitly load vfio_mdev driver.
+ */
+ request_module_nowait("vfio_mdev");
+
+ return ret;
}
static void __exit mdev_exit(void)
diff --git a/drivers/vfio/mdev/vfio_mdev.c b/drivers/vfio/mdev/vfio_mdev.c
new file mode 100644
index 000000000000..ffc36758cb84
--- /dev/null
+++ b/drivers/vfio/mdev/vfio_mdev.c
@@ -0,0 +1,148 @@
+/*
+ * VFIO based driver for Mediated device
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ * Author: Neo Jia <cjia@nvidia.com>
+ * Kirti Wankhede <kwankhede@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/vfio.h>
+#include <linux/mdev.h>
+
+#include "mdev_private.h"
+
+#define DRIVER_VERSION "0.1"
+#define DRIVER_AUTHOR "NVIDIA Corporation"
+#define DRIVER_DESC "VFIO based driver for Mediated device"
+
+static int vfio_mdev_open(void *device_data)
+{
+ struct mdev_device *mdev = device_data;
+ struct parent_device *parent = mdev->parent;
+ int ret;
+
+ if (unlikely(!parent->ops->open))
+ return -EINVAL;
+
+ if (!try_module_get(THIS_MODULE))
+ return -ENODEV;
+
+ ret = parent->ops->open(mdev);
+ if (ret)
+ module_put(THIS_MODULE);
+
+ return ret;
+}
+
+static void vfio_mdev_release(void *device_data)
+{
+ struct mdev_device *mdev = device_data;
+ struct parent_device *parent = mdev->parent;
+
+ if (likely(parent->ops->release))
+ parent->ops->release(mdev);
+
+ module_put(THIS_MODULE);
+}
+
+static long vfio_mdev_unlocked_ioctl(void *device_data,
+ unsigned int cmd, unsigned long arg)
+{
+ struct mdev_device *mdev = device_data;
+ struct parent_device *parent = mdev->parent;
+
+ if (unlikely(!parent->ops->ioctl))
+ return -EINVAL;
+
+ return parent->ops->ioctl(mdev, cmd, arg);
+}
+
+static ssize_t vfio_mdev_read(void *device_data, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct mdev_device *mdev = device_data;
+ struct parent_device *parent = mdev->parent;
+
+ if (unlikely(!parent->ops->read))
+ return -EINVAL;
+
+ return parent->ops->read(mdev, buf, count, ppos);
+}
+
+static ssize_t vfio_mdev_write(void *device_data, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct mdev_device *mdev = device_data;
+ struct parent_device *parent = mdev->parent;
+
+ if (unlikely(!parent->ops->write))
+ return -EINVAL;
+
+ return parent->ops->write(mdev, buf, count, ppos);
+}
+
+static int vfio_mdev_mmap(void *device_data, struct vm_area_struct *vma)
+{
+ struct mdev_device *mdev = device_data;
+ struct parent_device *parent = mdev->parent;
+
+ if (unlikely(!parent->ops->mmap))
+ return -EINVAL;
+
+ return parent->ops->mmap(mdev, vma);
+}
+
+static const struct vfio_device_ops vfio_mdev_dev_ops = {
+ .name = "vfio-mdev",
+ .open = vfio_mdev_open,
+ .release = vfio_mdev_release,
+ .ioctl = vfio_mdev_unlocked_ioctl,
+ .read = vfio_mdev_read,
+ .write = vfio_mdev_write,
+ .mmap = vfio_mdev_mmap,
+};
+
+int vfio_mdev_probe(struct device *dev)
+{
+ struct mdev_device *mdev = to_mdev_device(dev);
+
+ return vfio_add_group_dev(dev, &vfio_mdev_dev_ops, mdev);
+}
+
+void vfio_mdev_remove(struct device *dev)
+{
+ vfio_del_group_dev(dev);
+}
+
+struct mdev_driver vfio_mdev_driver = {
+ .name = "vfio_mdev",
+ .probe = vfio_mdev_probe,
+ .remove = vfio_mdev_remove,
+};
+
+static int __init vfio_mdev_init(void)
+{
+ return mdev_register_driver(&vfio_mdev_driver, THIS_MODULE);
+}
+
+static void __exit vfio_mdev_exit(void)
+{
+ mdev_unregister_driver(&vfio_mdev_driver);
+}
+
+module_init(vfio_mdev_init)
+module_exit(vfio_mdev_exit)
+
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
--
2.7.0
^ permalink raw reply related
* [Qemu-devel] [PATCH v12 12/22] vfio: Add notifier callback to parent's ops structure of mdev
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Add a notifier calback to parent's ops structure of mdev device so that per
device notifer for vfio module is registered through vfio_mdev module.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: Iafa6f1721aecdd6e50eb93b153b5621e6d29b637
---
drivers/vfio/mdev/vfio_mdev.c | 19 +++++++++++++++++++
include/linux/mdev.h | 9 +++++++++
2 files changed, 28 insertions(+)
diff --git a/drivers/vfio/mdev/vfio_mdev.c b/drivers/vfio/mdev/vfio_mdev.c
index ffc36758cb84..1694b1635607 100644
--- a/drivers/vfio/mdev/vfio_mdev.c
+++ b/drivers/vfio/mdev/vfio_mdev.c
@@ -24,6 +24,15 @@
#define DRIVER_AUTHOR "NVIDIA Corporation"
#define DRIVER_DESC "VFIO based driver for Mediated device"
+static int vfio_mdev_notifier(struct notifier_block *nb, unsigned long action,
+ void *data)
+{
+ struct mdev_device *mdev = container_of(nb, struct mdev_device, nb);
+ struct parent_device *parent = mdev->parent;
+
+ return parent->ops->notifier(mdev, action, data);
+}
+
static int vfio_mdev_open(void *device_data)
{
struct mdev_device *mdev = device_data;
@@ -40,6 +49,11 @@ static int vfio_mdev_open(void *device_data)
if (ret)
module_put(THIS_MODULE);
+ if (likely(parent->ops->notifier)) {
+ mdev->nb.notifier_call = vfio_mdev_notifier;
+ if (vfio_register_notifier(&mdev->dev, &mdev->nb))
+ pr_err("Failed to register notifier for mdev\n");
+ }
return ret;
}
@@ -48,6 +62,11 @@ static void vfio_mdev_release(void *device_data)
struct mdev_device *mdev = device_data;
struct parent_device *parent = mdev->parent;
+ if (likely(parent->ops->notifier)) {
+ if (vfio_unregister_notifier(&mdev->dev, &mdev->nb))
+ pr_err("Failed to unregister notifier for mdev\n");
+ }
+
if (likely(parent->ops->release))
parent->ops->release(mdev);
diff --git a/include/linux/mdev.h b/include/linux/mdev.h
index 4900cc472364..665afe0a4c31 100644
--- a/include/linux/mdev.h
+++ b/include/linux/mdev.h
@@ -37,6 +37,7 @@ struct mdev_device {
struct kref ref;
struct list_head next;
struct kobject *type_kobj;
+ struct notifier_block nb;
};
/**
@@ -85,6 +86,12 @@ struct mdev_device {
* @mmap: mmap callback
* @mdev: mediated device structure
* @vma: vma structure
+ * @notifer: Notifier callback, currently only for
+ * VFIO_IOMMU_NOTIFY_DMA_UNMAP action notified duing
+ * DMA_UNMAP call on mapped iova range.
+ * @mdev: mediated device structure
+ * @action: Action for which notifier is called
+ * @data: Data associated with the notifier
* Parent device that support mediated device should be registered with mdev
* module with parent_ops structure.
**/
@@ -106,6 +113,8 @@ struct parent_ops {
ssize_t (*ioctl)(struct mdev_device *mdev, unsigned int cmd,
unsigned long arg);
int (*mmap)(struct mdev_device *mdev, struct vm_area_struct *vma);
+ int (*notifier)(struct mdev_device *mdev, unsigned long action,
+ void *data);
};
/* interface for exporting mdev supported type attributes */
--
2.7.0
^ permalink raw reply related
* [PATCH next] dctcp: update cwnd on congestion event
From: Florian Westphal @ 2016-11-14 15:42 UTC (permalink / raw)
To: netdev
Cc: Florian Westphal, Lawrence Brakmo, Andrew Shewmaker, Glenn Judd,
Daniel Borkmann
draft-ietf-tcpm-dctcp-02 says:
... when the sender receives an indication of congestion
(ECE), the sender SHOULD update cwnd as follows:
cwnd = cwnd * (1 - DCTCP.Alpha / 2)
So, lets do this and reduce cwnd more smoothly (and faster), as per
current congestion estimate.
Cc: Lawrence Brakmo <brakmo@fb.com>
Cc: Andrew Shewmaker <agshew@gmail.com>
Cc: Glenn Judd <glenn.judd@morganstanley.com>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Florian Westphal <fw@strlen.de>
---
diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c
index ab37c6775630..51139175bf61 100644
--- a/net/ipv4/tcp_dctcp.c
+++ b/net/ipv4/tcp_dctcp.c
@@ -188,8 +188,8 @@ static void dctcp_ce_state_1_to_0(struct sock *sk)
static void dctcp_update_alpha(struct sock *sk, u32 flags)
{
- const struct tcp_sock *tp = tcp_sk(sk);
struct dctcp *ca = inet_csk_ca(sk);
+ struct tcp_sock *tp = tcp_sk(sk);
u32 acked_bytes = tp->snd_una - ca->prior_snd_una;
/* If ack did not advance snd_una, count dupack as MSS size.
@@ -229,6 +229,13 @@ static void dctcp_update_alpha(struct sock *sk, u32 flags)
WRITE_ONCE(ca->dctcp_alpha, alpha);
dctcp_reset(tp, ca);
}
+
+ if (flags & CA_ACK_ECE) {
+ unsigned int cwnd = dctcp_ssthresh(sk);
+
+ if (cwnd != tp->snd_cwnd)
+ tp->snd_cwnd = cwnd;
+ }
}
static void dctcp_state(struct sock *sk, u8 new_state)
--
2.7.3
^ permalink raw reply related
* [Qemu-devel] [PATCH v12 11/22] vfio iommu: Add blocking notifier to notify DMA_UNMAP
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Added blocking notifier to IOMMU TYPE1 driver to notify vendor drivers
about DMA_UNMAP.
Exported two APIs vfio_register_notifier() and vfio_unregister_notifier().
Notifier should be registered, if external user wants to use
vfio_pin_pages()/vfio_unpin_pages() APIs to pin/unpin pages.
Vendor driver should use VFIO_IOMMU_NOTIFY_DMA_UNMAP action to invalidate
mappings.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I5910d0024d6be87f3e8d3e0ca0eaeaaa0b17f271
---
drivers/vfio/vfio.c | 73 +++++++++++++++++++++++++++++++++++++++++
drivers/vfio/vfio_iommu_type1.c | 60 ++++++++++++++++++++++++++-------
include/linux/vfio.h | 11 +++++++
3 files changed, 132 insertions(+), 12 deletions(-)
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 7dcfbca2016a..388a3cbddbd9 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -1902,6 +1902,79 @@ err_unpin_pages:
}
EXPORT_SYMBOL(vfio_unpin_pages);
+int vfio_register_notifier(struct device *dev, struct notifier_block *nb)
+{
+ struct vfio_container *container;
+ struct vfio_group *group;
+ struct vfio_iommu_driver *driver;
+ ssize_t ret;
+
+ if (!dev || !nb)
+ return -EINVAL;
+
+ group = vfio_group_get_from_dev(dev);
+ if (IS_ERR(group))
+ return PTR_ERR(group);
+
+ ret = vfio_group_add_container_user(group);
+ if (ret)
+ goto err_register_nb;
+
+ container = group->container;
+ down_read(&container->group_lock);
+
+ driver = container->iommu_driver;
+ if (likely(driver && driver->ops->register_notifier))
+ ret = driver->ops->register_notifier(container->iommu_data, nb);
+ else
+ ret = -ENOTTY;
+
+ up_read(&container->group_lock);
+ vfio_group_try_dissolve_container(group);
+
+err_register_nb:
+ vfio_group_put(group);
+ return ret;
+}
+EXPORT_SYMBOL(vfio_register_notifier);
+
+int vfio_unregister_notifier(struct device *dev, struct notifier_block *nb)
+{
+ struct vfio_container *container;
+ struct vfio_group *group;
+ struct vfio_iommu_driver *driver;
+ ssize_t ret;
+
+ if (!dev || !nb)
+ return -EINVAL;
+
+ group = vfio_group_get_from_dev(dev);
+ if (IS_ERR(group))
+ return PTR_ERR(group);
+
+ ret = vfio_group_add_container_user(group);
+ if (ret)
+ goto err_unregister_nb;
+
+ container = group->container;
+ down_read(&container->group_lock);
+
+ driver = container->iommu_driver;
+ if (likely(driver && driver->ops->unregister_notifier))
+ ret = driver->ops->unregister_notifier(container->iommu_data,
+ nb);
+ else
+ ret = -ENOTTY;
+
+ up_read(&container->group_lock);
+ vfio_group_try_dissolve_container(group);
+
+err_unregister_nb:
+ vfio_group_put(group);
+ return ret;
+}
+EXPORT_SYMBOL(vfio_unregister_notifier);
+
/**
* Module/class support
*/
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 2697d874dd35..7e034e7c5ea5 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -38,6 +38,7 @@
#include <linux/workqueue.h>
#include <linux/pid_namespace.h>
#include <linux/mdev.h>
+#include <linux/notifier.h>
#define DRIVER_VERSION "0.2"
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
@@ -60,6 +61,7 @@ struct vfio_iommu {
struct vfio_domain *external_domain; /* domain for external user */
struct mutex lock;
struct rb_root dma_list;
+ struct blocking_notifier_head notifier;
bool v2;
bool nesting;
};
@@ -568,7 +570,8 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
mutex_lock(&iommu->lock);
- if (!iommu->external_domain) {
+ /* Fail if notifier list is empty */
+ if ((!iommu->external_domain) || (!iommu->notifier.head)) {
ret = -EINVAL;
goto pin_done;
}
@@ -854,15 +857,29 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
if (dma->task->mm != current->mm)
break;
unmapped += dma->size;
+
+ mutex_unlock(&iommu->lock);
+ if (iommu->external_domain && !RB_EMPTY_ROOT(&dma->pfn_list)) {
+ struct vfio_iommu_type1_dma_unmap nb_unmap;
+
+ nb_unmap.iova = dma->iova;
+ nb_unmap.size = dma->size;
+ blocking_notifier_call_chain(&iommu->notifier,
+ VFIO_IOMMU_NOTIFY_DMA_UNMAP,
+ &nb_unmap);
+
+ if (WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)))
+ goto unmap_exit;
+ }
+ mutex_lock(&iommu->lock);
vfio_remove_dma(iommu, dma);
}
unlock:
mutex_unlock(&iommu->lock);
-
+unmap_exit:
/* Report how much was unmapped */
unmap->size = unmapped;
-
return ret;
}
@@ -1430,6 +1447,7 @@ static void *vfio_iommu_type1_open(unsigned long arg)
INIT_LIST_HEAD(&iommu->domain_list);
iommu->dma_list = RB_ROOT;
mutex_init(&iommu->lock);
+ BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier);
return iommu;
}
@@ -1565,16 +1583,34 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
return -ENOTTY;
}
+static int vfio_iommu_type1_register_notifier(void *iommu_data,
+ struct notifier_block *nb)
+{
+ struct vfio_iommu *iommu = iommu_data;
+
+ return blocking_notifier_chain_register(&iommu->notifier, nb);
+}
+
+static int vfio_iommu_type1_unregister_notifier(void *iommu_data,
+ struct notifier_block *nb)
+{
+ struct vfio_iommu *iommu = iommu_data;
+
+ return blocking_notifier_chain_unregister(&iommu->notifier, nb);
+}
+
static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
- .name = "vfio-iommu-type1",
- .owner = THIS_MODULE,
- .open = vfio_iommu_type1_open,
- .release = vfio_iommu_type1_release,
- .ioctl = vfio_iommu_type1_ioctl,
- .attach_group = vfio_iommu_type1_attach_group,
- .detach_group = vfio_iommu_type1_detach_group,
- .pin_pages = vfio_iommu_type1_pin_pages,
- .unpin_pages = vfio_iommu_type1_unpin_pages,
+ .name = "vfio-iommu-type1",
+ .owner = THIS_MODULE,
+ .open = vfio_iommu_type1_open,
+ .release = vfio_iommu_type1_release,
+ .ioctl = vfio_iommu_type1_ioctl,
+ .attach_group = vfio_iommu_type1_attach_group,
+ .detach_group = vfio_iommu_type1_detach_group,
+ .pin_pages = vfio_iommu_type1_pin_pages,
+ .unpin_pages = vfio_iommu_type1_unpin_pages,
+ .register_notifier = vfio_iommu_type1_register_notifier,
+ .unregister_notifier = vfio_iommu_type1_unregister_notifier,
};
static int __init vfio_iommu_type1_init(void)
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 86f507d0f585..123e089388e9 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -80,6 +80,10 @@ struct vfio_iommu_driver_ops {
unsigned long *phys_pfn);
int (*unpin_pages)(void *iommu_data,
unsigned long *user_pfn, int npage);
+ int (*register_notifier)(void *iommu_data,
+ struct notifier_block *nb);
+ int (*unregister_notifier)(void *iommu_data,
+ struct notifier_block *nb);
};
extern int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops);
@@ -137,6 +141,13 @@ extern int vfio_pin_pages(struct device *dev, unsigned long *user_pfn,
extern int vfio_unpin_pages(struct device *dev, unsigned long *user_pfn,
int npage);
+#define VFIO_IOMMU_NOTIFY_DMA_UNMAP 1
+
+extern int vfio_register_notifier(struct device *dev,
+ struct notifier_block *nb);
+
+extern int vfio_unregister_notifier(struct device *dev,
+ struct notifier_block *nb);
/*
* IRQfd - generic
*/
--
2.7.0
^ permalink raw reply related
* [Qemu-devel] [PATCH v12 10/22] vfio iommu type1: Add support for mediated devices
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
VFIO IOMMU drivers are designed for the devices which are IOMMU capable.
Mediated device only uses IOMMU APIs, the underlying hardware can be
managed by an IOMMU domain.
Aim of this change is:
- To use most of the code of TYPE1 IOMMU driver for mediated devices
- To support direct assigned device and mediated device in single module
This change adds pin and unpin support for mediated device to TYPE1 IOMMU
backend module. More details:
- Domain for external user is tracked separately in vfio_iommu structure.
It is allocated when group for first mdev device is attached.
- Pages pinned for external domain are tracked in each vfio_dma structure
for that iova range.
- Page tracking rb-tree in vfio_dma keeps <iova, pfn, ref_count>. Key of
rb-tree is iova, but it actually aims to track pfns.
- On external pin request for an iova, page is pinned only once, if iova
is already pinned and tracked, ref_count is incremented.
- External unin request unpins pages only when ref_count is 0.
- Pinned pages list is used to verify unpinning request and to unpin
remaining pages while detaching the group for that device.
- Page accounting is updated to account in its address space where the
pages are pinned/unpinned, i.e dma->task
- Accouting for mdev device is only done if there is no iommu capable
domain in the container. When there is a direct device assigned to the
container and that domain is iommu capable, all pages are already pinned
during DMA_MAP.
- Page accouting is updated on hot plug and unplug mdev device and pass
through device.
Tested by assigning below combinations of devices to a single VM:
- GPU pass through only
- vGPU device only
- One GPU pass through and one vGPU device
- Linux VM hot plug and unplug vGPU device while GPU pass through device
exist
- Linux VM hot plug and unplug GPU pass through device while vGPU device
exist
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I295d6f0f2e0579b8d9882bfd8fd5a4194b97bd9a
---
drivers/vfio/vfio_iommu_type1.c | 587 +++++++++++++++++++++++++++++++++++-----
1 file changed, 526 insertions(+), 61 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 50aca95cf61e..2697d874dd35 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -37,6 +37,7 @@
#include <linux/vfio.h>
#include <linux/workqueue.h>
#include <linux/pid_namespace.h>
+#include <linux/mdev.h>
#define DRIVER_VERSION "0.2"
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
@@ -56,6 +57,7 @@ MODULE_PARM_DESC(disable_hugepages,
struct vfio_iommu {
struct list_head domain_list;
+ struct vfio_domain *external_domain; /* domain for external user */
struct mutex lock;
struct rb_root dma_list;
bool v2;
@@ -76,7 +78,9 @@ struct vfio_dma {
unsigned long vaddr; /* Process virtual addr */
size_t size; /* Map size (bytes) */
int prot; /* IOMMU_READ/WRITE */
+ bool iommu_mapped;
struct task_struct *task;
+ struct rb_root pfn_list; /* Ex-user pinned pfn list */
};
struct vfio_group {
@@ -85,6 +89,21 @@ struct vfio_group {
};
/*
+ * Guest RAM pinning working set or DMA target
+ */
+struct vfio_pfn {
+ struct rb_node node;
+ dma_addr_t iova; /* Device address */
+ unsigned long pfn; /* Host pfn */
+ atomic_t ref_count;
+};
+
+#define IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu) \
+ (!list_empty(&iommu->domain_list))
+
+static int put_pfn(unsigned long pfn, int prot);
+
+/*
* This code handles mapping and unmapping of user data buffers
* into DMA'ble space using the IOMMU
*/
@@ -132,6 +151,97 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
rb_erase(&old->node, &iommu->dma_list);
}
+/*
+ * Helper Functions for host iova-pfn list
+ */
+static struct vfio_pfn *vfio_find_vpfn(struct vfio_dma *dma, dma_addr_t iova)
+{
+ struct vfio_pfn *vpfn;
+ struct rb_node *node = dma->pfn_list.rb_node;
+
+ while (node) {
+ vpfn = rb_entry(node, struct vfio_pfn, node);
+
+ if (iova < vpfn->iova)
+ node = node->rb_left;
+ else if (iova > vpfn->iova)
+ node = node->rb_right;
+ else
+ return vpfn;
+ }
+ return NULL;
+}
+
+static void vfio_link_pfn(struct vfio_dma *dma,
+ struct vfio_pfn *new)
+{
+ struct rb_node **link, *parent = NULL;
+ struct vfio_pfn *vpfn;
+
+ link = &dma->pfn_list.rb_node;
+ while (*link) {
+ parent = *link;
+ vpfn = rb_entry(parent, struct vfio_pfn, node);
+
+ if (new->iova < vpfn->iova)
+ link = &(*link)->rb_left;
+ else
+ link = &(*link)->rb_right;
+ }
+
+ rb_link_node(&new->node, parent, link);
+ rb_insert_color(&new->node, &dma->pfn_list);
+}
+
+static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old)
+{
+ rb_erase(&old->node, &dma->pfn_list);
+}
+
+static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova,
+ unsigned long pfn)
+{
+ struct vfio_pfn *vpfn;
+
+ vpfn = kzalloc(sizeof(*vpfn), GFP_KERNEL);
+ if (!vpfn)
+ return -ENOMEM;
+
+ vpfn->iova = iova;
+ vpfn->pfn = pfn;
+ atomic_set(&vpfn->ref_count, 1);
+ vfio_link_pfn(dma, vpfn);
+ return 0;
+}
+
+static void vfio_remove_from_pfn_list(struct vfio_dma *dma,
+ struct vfio_pfn *vpfn)
+{
+ vfio_unlink_pfn(dma, vpfn);
+ kfree(vpfn);
+}
+
+static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma,
+ unsigned long iova)
+{
+ struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
+
+ if (vpfn)
+ atomic_inc(&vpfn->ref_count);
+ return vpfn;
+}
+
+static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn)
+{
+ int ret = 0;
+
+ if (atomic_dec_and_test(&vpfn->ref_count)) {
+ ret = put_pfn(vpfn->pfn, dma->prot);
+ vfio_remove_from_pfn_list(dma, vpfn);
+ }
+ return ret;
+}
+
struct vwork {
struct mm_struct *mm;
long npage;
@@ -270,7 +380,6 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
}
up_read(&mm->mmap_sem);
-
return ret;
}
@@ -280,28 +389,35 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
* first page and all consecutive pages with the same locking.
*/
static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
- long npage, int prot, unsigned long *pfn_base)
+ long npage, unsigned long *pfn_base)
{
unsigned long limit;
bool lock_cap = ns_capable(task_active_pid_ns(dma->task)->user_ns,
CAP_IPC_LOCK);
struct mm_struct *mm;
- long ret, i;
+ long ret, i, lock_acct = 0;
bool rsvd;
+ struct vfio_pfn *vpfn;
+ dma_addr_t iova;
mm = get_task_mm(dma->task);
if (!mm)
return -ENODEV;
- ret = vaddr_get_pfn(mm, vaddr, prot, pfn_base);
+ ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);
if (ret)
goto pin_pg_remote_exit;
+ iova = vaddr - dma->vaddr + dma->iova;
+ vpfn = vfio_find_vpfn(dma, iova);
+ if (!vpfn)
+ lock_acct = 1;
+
rsvd = is_invalid_reserved_pfn(*pfn_base);
limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
- if (!rsvd && !lock_cap && mm->locked_vm + 1 > limit) {
- put_pfn(*pfn_base, prot);
+ if (!rsvd && !lock_cap && mm->locked_vm + lock_acct > limit) {
+ put_pfn(*pfn_base, dma->prot);
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
limit << PAGE_SHIFT);
ret = -ENOMEM;
@@ -310,7 +426,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
if (unlikely(disable_hugepages)) {
if (!rsvd)
- vfio_lock_acct(dma->task, 1);
+ vfio_lock_acct(dma->task, lock_acct);
ret = 1;
goto pin_pg_remote_exit;
}
@@ -319,18 +435,24 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
unsigned long pfn = 0;
- ret = vaddr_get_pfn(mm, vaddr, prot, &pfn);
+ ret = vaddr_get_pfn(mm, vaddr, dma->prot, &pfn);
if (ret)
break;
if (pfn != *pfn_base + i ||
rsvd != is_invalid_reserved_pfn(pfn)) {
- put_pfn(pfn, prot);
+ put_pfn(pfn, dma->prot);
break;
}
- if (!rsvd && !lock_cap && mm->locked_vm + i + 1 > limit) {
- put_pfn(pfn, prot);
+ iova = vaddr - dma->vaddr + dma->iova;
+ vpfn = vfio_find_vpfn(dma, iova);
+ if (!vpfn)
+ lock_acct++;
+
+ if (!rsvd && !lock_cap &&
+ mm->locked_vm + lock_acct + 1 > limit) {
+ put_pfn(pfn, dma->prot);
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
__func__, limit << PAGE_SHIFT);
break;
@@ -338,7 +460,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
}
if (!rsvd)
- vfio_lock_acct(dma->task, i);
+ vfio_lock_acct(dma->task, lock_acct);
ret = i;
pin_pg_remote_exit:
@@ -346,14 +468,78 @@ pin_pg_remote_exit:
return ret;
}
-static long vfio_unpin_pages_remote(struct vfio_dma *dma, unsigned long pfn,
- long npage, int prot, bool do_accounting)
+static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
+ unsigned long pfn, long npage,
+ bool do_accounting)
{
- unsigned long unlocked = 0;
+ long unlocked = 0, locked = 0;
long i;
- for (i = 0; i < npage; i++)
- unlocked += put_pfn(pfn++, prot);
+ for (i = 0; i < npage; i++) {
+ struct vfio_pfn *vpfn;
+
+ unlocked += put_pfn(pfn++, dma->prot);
+
+ vpfn = vfio_find_vpfn(dma, iova + (i << PAGE_SHIFT));
+ if (vpfn)
+ locked++;
+ }
+
+ if (do_accounting)
+ vfio_lock_acct(dma->task, locked - unlocked);
+
+ return unlocked;
+}
+
+static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
+ unsigned long *pfn_base, bool do_accounting)
+{
+ unsigned long limit;
+ bool lock_cap = ns_capable(task_active_pid_ns(dma->task)->user_ns,
+ CAP_IPC_LOCK);
+ struct mm_struct *mm;
+ int ret;
+ bool rsvd;
+
+ mm = get_task_mm(dma->task);
+ if (!mm)
+ return -ENODEV;
+
+ ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);
+ if (ret)
+ goto pin_page_exit;
+
+ rsvd = is_invalid_reserved_pfn(*pfn_base);
+ limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+
+ if (!rsvd && !lock_cap && mm->locked_vm + 1 > limit) {
+ put_pfn(*pfn_base, dma->prot);
+ pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK (%ld) exceeded\n",
+ __func__, dma->task->comm, task_pid_nr(dma->task),
+ limit << PAGE_SHIFT);
+ ret = -ENOMEM;
+ goto pin_page_exit;
+ }
+
+ if (!rsvd && do_accounting)
+ vfio_lock_acct(dma->task, 1);
+ ret = 1;
+
+pin_page_exit:
+ mmput(mm);
+ return ret;
+}
+
+static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova,
+ bool do_accounting)
+{
+ int unlocked;
+ struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
+
+ if (!vpfn)
+ return 0;
+
+ unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);
if (do_accounting)
vfio_lock_acct(dma->task, -unlocked);
@@ -361,14 +547,148 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, unsigned long pfn,
return unlocked;
}
-static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
+static int vfio_iommu_type1_pin_pages(void *iommu_data,
+ unsigned long *user_pfn,
+ int npage, int prot,
+ unsigned long *phys_pfn)
+{
+ struct vfio_iommu *iommu = iommu_data;
+ int i, j, ret;
+ unsigned long remote_vaddr;
+ unsigned long *pfn = phys_pfn;
+ struct vfio_dma *dma;
+ bool do_accounting;
+
+ if (!iommu || !user_pfn || !phys_pfn)
+ return -EINVAL;
+
+ /* Supported for v2 version only */
+ if (!iommu->v2)
+ return -EACCES;
+
+ mutex_lock(&iommu->lock);
+
+ if (!iommu->external_domain) {
+ ret = -EINVAL;
+ goto pin_done;
+ }
+
+ /*
+ * If iommu capable domain exist in the container then all pages are
+ * already pinned and accounted. Accouting should be done if there is no
+ * iommu capable domain in the container.
+ */
+ do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
+
+ for (i = 0; i < npage; i++) {
+ dma_addr_t iova;
+ struct vfio_pfn *vpfn;
+
+ iova = user_pfn[i] << PAGE_SHIFT;
+ dma = vfio_find_dma(iommu, iova, 0);
+ if (!dma) {
+ ret = -EINVAL;
+ goto pin_unwind;
+ }
+
+ if ((dma->prot & prot) != prot) {
+ pr_info("%s: dma->prot: 0x%x prot: 0x%x\n",
+ __func__, dma->prot, prot);
+ ret = -EPERM;
+ goto pin_unwind;
+ }
+
+ vpfn = vfio_iova_get_vfio_pfn(dma, iova);
+ if (vpfn) {
+ pfn[i] = vpfn->pfn;
+ continue;
+ }
+
+ remote_vaddr = dma->vaddr + iova - dma->iova;
+ ret = vfio_pin_page_external(dma, remote_vaddr, &pfn[i],
+ do_accounting);
+ if (ret <= 0) {
+ WARN_ON(!ret);
+ goto pin_unwind;
+ }
+
+ ret = vfio_add_to_pfn_list(dma, iova, pfn[i]);
+ if (ret) {
+ vfio_unpin_page_external(dma, iova, do_accounting);
+ goto pin_unwind;
+ }
+ }
+
+ ret = i;
+ goto pin_done;
+
+pin_unwind:
+ pfn[i] = 0;
+ for (j = 0; j < i; j++) {
+ dma_addr_t iova;
+
+ iova = user_pfn[j] << PAGE_SHIFT;
+ dma = vfio_find_dma(iommu, iova, 0);
+ vfio_unpin_page_external(dma, iova, do_accounting);
+ pfn[j] = 0;
+ }
+pin_done:
+ mutex_unlock(&iommu->lock);
+ return ret;
+}
+
+static int vfio_iommu_type1_unpin_pages(void *iommu_data,
+ unsigned long *user_pfn,
+ int npage)
+{
+ struct vfio_iommu *iommu = iommu_data;
+ bool do_accounting;
+ int unlocked = 0, i;
+
+ if (!iommu || !user_pfn)
+ return -EINVAL;
+
+ /* Supported for v2 version only */
+ if (!iommu->v2)
+ return -EACCES;
+
+ mutex_lock(&iommu->lock);
+
+ if (!iommu->external_domain) {
+ mutex_unlock(&iommu->lock);
+ return -EINVAL;
+ }
+
+ do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
+ for (i = 0; i < npage; i++) {
+ struct vfio_dma *dma;
+ dma_addr_t iova;
+
+ iova = user_pfn[i] << PAGE_SHIFT;
+ dma = vfio_find_dma(iommu, iova, 0);
+ if (!dma)
+ goto unpin_exit;
+ unlocked += vfio_unpin_page_external(dma, iova, do_accounting);
+ }
+
+unpin_exit:
+ mutex_unlock(&iommu->lock);
+ return unlocked;
+}
+
+static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
+ bool do_accounting)
{
dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
struct vfio_domain *domain, *d;
long unlocked = 0;
if (!dma->size)
- return;
+ return 0;
+
+ if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
+ return 0;
+
/*
* We use the IOMMU to track the physical addresses, otherwise we'd
* need a much more complicated tracking system. Unfortunately that
@@ -410,20 +730,26 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
if (WARN_ON(!unmapped))
break;
- unlocked += vfio_unpin_pages_remote(dma, phys >> PAGE_SHIFT,
+ unlocked += vfio_unpin_pages_remote(dma, iova,
+ phys >> PAGE_SHIFT,
unmapped >> PAGE_SHIFT,
- dma->prot, false);
+ false);
iova += unmapped;
cond_resched();
}
- vfio_lock_acct(dma->task, -unlocked);
+ dma->iommu_mapped = false;
+ if (do_accounting) {
+ vfio_lock_acct(dma->task, -unlocked);
+ return 0;
+ }
+ return unlocked;
}
static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
{
- vfio_unmap_unpin(iommu, dma);
+ vfio_unmap_unpin(iommu, dma, true);
vfio_unlink_dma(iommu, dma);
put_task_struct(dma->task);
kfree(dma);
@@ -606,8 +932,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
while (size) {
/* Pin a contiguous chunk of memory */
npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
- size >> PAGE_SHIFT, dma->prot,
- &pfn);
+ size >> PAGE_SHIFT, &pfn);
if (npage <= 0) {
WARN_ON(!npage);
ret = (int)npage;
@@ -618,8 +943,8 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,
dma->prot);
if (ret) {
- vfio_unpin_pages_remote(dma, pfn, npage,
- dma->prot, true);
+ vfio_unpin_pages_remote(dma, iova + dma->size, pfn,
+ npage, true);
break;
}
@@ -627,6 +952,8 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
dma->size += npage << PAGE_SHIFT;
}
+ dma->iommu_mapped = true;
+
if (ret)
vfio_remove_dma(iommu, dma);
@@ -682,11 +1009,16 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
dma->prot = prot;
get_task_struct(current);
dma->task = current;
+ dma->pfn_list = RB_ROOT;
/* Insert zero-sized and grow as we map chunks of it */
vfio_link_dma(iommu, dma);
- ret = vfio_pin_map_dma(iommu, dma, size);
+ /* Don't pin and map if container doesn't contain IOMMU capable domain*/
+ if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
+ dma->size = size;
+ else
+ ret = vfio_pin_map_dma(iommu, dma, size);
do_map_err:
mutex_unlock(&iommu->lock);
return ret;
@@ -715,10 +1047,6 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
d = list_first_entry(&iommu->domain_list, struct vfio_domain, next);
n = rb_first(&iommu->dma_list);
- /* If there's not a domain, there better not be any mappings */
- if (WARN_ON(n && !d))
- return -EINVAL;
-
for (; n; n = rb_next(n)) {
struct vfio_dma *dma;
dma_addr_t iova;
@@ -727,21 +1055,49 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
iova = dma->iova;
while (iova < dma->iova + dma->size) {
- phys_addr_t phys = iommu_iova_to_phys(d->domain, iova);
+ phys_addr_t phys;
size_t size;
- if (WARN_ON(!phys)) {
- iova += PAGE_SIZE;
- continue;
+ if (dma->iommu_mapped) {
+ phys_addr_t p;
+ dma_addr_t i;
+
+ phys = iommu_iova_to_phys(d->domain, iova);
+
+ if (WARN_ON(!phys)) {
+ iova += PAGE_SIZE;
+ continue;
+ }
+
+ size = PAGE_SIZE;
+ p = phys + size;
+ i = iova + size;
+ while (i < dma->iova + dma->size &&
+ p == iommu_iova_to_phys(d->domain, i)) {
+ size += PAGE_SIZE;
+ p += PAGE_SIZE;
+ i += PAGE_SIZE;
+ }
+ } else {
+ unsigned long pfn;
+ unsigned long vaddr = dma->vaddr +
+ (iova - dma->iova);
+ size_t n = dma->iova + dma->size - iova;
+ long npage;
+
+ npage = vfio_pin_pages_remote(dma, vaddr,
+ n >> PAGE_SHIFT,
+ &pfn);
+ if (npage <= 0) {
+ WARN_ON(!npage);
+ ret = (int)npage;
+ return ret;
+ }
+
+ phys = pfn << PAGE_SHIFT;
+ size = npage << PAGE_SHIFT;
}
- size = PAGE_SIZE;
-
- while (iova + size < dma->iova + dma->size &&
- phys + size == iommu_iova_to_phys(d->domain,
- iova + size))
- size += PAGE_SIZE;
-
ret = iommu_map(domain->domain, iova, phys,
size, dma->prot | domain->prot);
if (ret)
@@ -749,8 +1105,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
iova += size;
}
+ dma->iommu_mapped = true;
}
-
return 0;
}
@@ -806,7 +1162,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
struct vfio_iommu *iommu = iommu_data;
struct vfio_group *group;
struct vfio_domain *domain, *d;
- struct bus_type *bus = NULL;
+ struct bus_type *bus = NULL, *mdev_bus;
int ret;
mutex_lock(&iommu->lock);
@@ -818,6 +1174,13 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
}
}
+ if (iommu->external_domain) {
+ if (find_iommu_group(iommu->external_domain, iommu_group)) {
+ mutex_unlock(&iommu->lock);
+ return -EINVAL;
+ }
+ }
+
group = kzalloc(sizeof(*group), GFP_KERNEL);
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!group || !domain) {
@@ -832,6 +1195,25 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
if (ret)
goto out_free;
+ mdev_bus = symbol_get(mdev_bus_type);
+
+ if (mdev_bus) {
+ if ((bus == mdev_bus) && !iommu_present(bus)) {
+ symbol_put(mdev_bus_type);
+ if (!iommu->external_domain) {
+ INIT_LIST_HEAD(&domain->group_list);
+ iommu->external_domain = domain;
+ } else
+ kfree(domain);
+
+ list_add(&group->next,
+ &iommu->external_domain->group_list);
+ mutex_unlock(&iommu->lock);
+ return 0;
+ }
+ symbol_put(mdev_bus_type);
+ }
+
domain->domain = iommu_domain_alloc(bus);
if (!domain->domain) {
ret = -EIO;
@@ -922,6 +1304,46 @@ static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu)
vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node));
}
+static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu)
+{
+ struct rb_node *n, *p;
+
+ n = rb_first(&iommu->dma_list);
+ for (; n; n = rb_next(n)) {
+ struct vfio_dma *dma;
+ long locked = 0, unlocked = 0;
+
+ dma = rb_entry(n, struct vfio_dma, node);
+ unlocked += vfio_unmap_unpin(iommu, dma, false);
+ p = rb_first(&dma->pfn_list);
+ for (; p; p = rb_next(p))
+ locked++;
+ vfio_lock_acct(dma->task, locked - unlocked);
+ }
+}
+
+static void vfio_external_unpin_all(struct vfio_iommu *iommu,
+ bool do_accounting)
+{
+ struct rb_node *n, *p;
+
+ n = rb_first(&iommu->dma_list);
+ for (; n; n = rb_next(n)) {
+ struct vfio_dma *dma;
+
+ dma = rb_entry(n, struct vfio_dma, node);
+ while ((p = rb_first(&dma->pfn_list))) {
+ int unlocked;
+ struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn,
+ node);
+
+ unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);
+ if (do_accounting)
+ vfio_lock_acct(dma->task, -unlocked);
+ }
+ }
+}
+
static void vfio_iommu_type1_detach_group(void *iommu_data,
struct iommu_group *iommu_group)
{
@@ -931,6 +1353,26 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
mutex_lock(&iommu->lock);
+ if (iommu->external_domain) {
+ domain = iommu->external_domain;
+ group = find_iommu_group(domain, iommu_group);
+ if (group) {
+ list_del(&group->next);
+ kfree(group);
+
+ if (list_empty(&domain->group_list)) {
+ if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) {
+ vfio_external_unpin_all(iommu, true);
+ vfio_iommu_unmap_unpin_all(iommu);
+ } else
+ vfio_external_unpin_all(iommu, false);
+ kfree(domain);
+ iommu->external_domain = NULL;
+ }
+ goto detach_group_done;
+ }
+ }
+
list_for_each_entry(domain, &iommu->domain_list, next) {
group = find_iommu_group(domain, iommu_group);
if (!group)
@@ -940,21 +1382,27 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
list_del(&group->next);
kfree(group);
/*
- * Group ownership provides privilege, if the group
- * list is empty, the domain goes away. If it's the
- * last domain, then all the mappings go away too.
+ * Group ownership provides privilege, if the group list is
+ * empty, the domain goes away. If it's the last domain with
+ * iommu and external domain doesn't exist, then all the
+ * mappings go away too. If it's the last domain with iommu and
+ * external domain exist, update accounting
*/
if (list_empty(&domain->group_list)) {
- if (list_is_singular(&iommu->domain_list))
- vfio_iommu_unmap_unpin_all(iommu);
+ if (list_is_singular(&iommu->domain_list)) {
+ if (!iommu->external_domain)
+ vfio_iommu_unmap_unpin_all(iommu);
+ else
+ vfio_iommu_unmap_unpin_reaccount(iommu);
+ }
iommu_domain_free(domain->domain);
list_del(&domain->next);
kfree(domain);
}
- goto done;
+ break;
}
-done:
+detach_group_done:
mutex_unlock(&iommu->lock);
}
@@ -986,27 +1434,42 @@ static void *vfio_iommu_type1_open(unsigned long arg)
return iommu;
}
+static void vfio_release_domain(struct vfio_domain *domain, bool external)
+{
+ struct vfio_group *group, *group_tmp;
+
+ list_for_each_entry_safe(group, group_tmp,
+ &domain->group_list, next) {
+ if (!external)
+ iommu_detach_group(domain->domain, group->iommu_group);
+ list_del(&group->next);
+ kfree(group);
+ }
+
+ if (!external)
+ iommu_domain_free(domain->domain);
+}
+
static void vfio_iommu_type1_release(void *iommu_data)
{
struct vfio_iommu *iommu = iommu_data;
struct vfio_domain *domain, *domain_tmp;
- struct vfio_group *group, *group_tmp;
+
+ if (iommu->external_domain) {
+ vfio_release_domain(iommu->external_domain, true);
+ vfio_external_unpin_all(iommu, false);
+ kfree(iommu->external_domain);
+ iommu->external_domain = NULL;
+ }
vfio_iommu_unmap_unpin_all(iommu);
list_for_each_entry_safe(domain, domain_tmp,
&iommu->domain_list, next) {
- list_for_each_entry_safe(group, group_tmp,
- &domain->group_list, next) {
- iommu_detach_group(domain->domain, group->iommu_group);
- list_del(&group->next);
- kfree(group);
- }
- iommu_domain_free(domain->domain);
+ vfio_release_domain(domain, false);
list_del(&domain->next);
kfree(domain);
}
-
kfree(iommu);
}
@@ -1110,6 +1573,8 @@ static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
.ioctl = vfio_iommu_type1_ioctl,
.attach_group = vfio_iommu_type1_attach_group,
.detach_group = vfio_iommu_type1_detach_group,
+ .pin_pages = vfio_iommu_type1_pin_pages,
+ .unpin_pages = vfio_iommu_type1_unpin_pages,
};
static int __init vfio_iommu_type1_init(void)
--
2.7.0
^ permalink raw reply related
* [PATCH v12 11/22] vfio iommu: Add blocking notifier to notify DMA_UNMAP
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Added blocking notifier to IOMMU TYPE1 driver to notify vendor drivers
about DMA_UNMAP.
Exported two APIs vfio_register_notifier() and vfio_unregister_notifier().
Notifier should be registered, if external user wants to use
vfio_pin_pages()/vfio_unpin_pages() APIs to pin/unpin pages.
Vendor driver should use VFIO_IOMMU_NOTIFY_DMA_UNMAP action to invalidate
mappings.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I5910d0024d6be87f3e8d3e0ca0eaeaaa0b17f271
---
drivers/vfio/vfio.c | 73 +++++++++++++++++++++++++++++++++++++++++
drivers/vfio/vfio_iommu_type1.c | 60 ++++++++++++++++++++++++++-------
include/linux/vfio.h | 11 +++++++
3 files changed, 132 insertions(+), 12 deletions(-)
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 7dcfbca2016a..388a3cbddbd9 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -1902,6 +1902,79 @@ err_unpin_pages:
}
EXPORT_SYMBOL(vfio_unpin_pages);
+int vfio_register_notifier(struct device *dev, struct notifier_block *nb)
+{
+ struct vfio_container *container;
+ struct vfio_group *group;
+ struct vfio_iommu_driver *driver;
+ ssize_t ret;
+
+ if (!dev || !nb)
+ return -EINVAL;
+
+ group = vfio_group_get_from_dev(dev);
+ if (IS_ERR(group))
+ return PTR_ERR(group);
+
+ ret = vfio_group_add_container_user(group);
+ if (ret)
+ goto err_register_nb;
+
+ container = group->container;
+ down_read(&container->group_lock);
+
+ driver = container->iommu_driver;
+ if (likely(driver && driver->ops->register_notifier))
+ ret = driver->ops->register_notifier(container->iommu_data, nb);
+ else
+ ret = -ENOTTY;
+
+ up_read(&container->group_lock);
+ vfio_group_try_dissolve_container(group);
+
+err_register_nb:
+ vfio_group_put(group);
+ return ret;
+}
+EXPORT_SYMBOL(vfio_register_notifier);
+
+int vfio_unregister_notifier(struct device *dev, struct notifier_block *nb)
+{
+ struct vfio_container *container;
+ struct vfio_group *group;
+ struct vfio_iommu_driver *driver;
+ ssize_t ret;
+
+ if (!dev || !nb)
+ return -EINVAL;
+
+ group = vfio_group_get_from_dev(dev);
+ if (IS_ERR(group))
+ return PTR_ERR(group);
+
+ ret = vfio_group_add_container_user(group);
+ if (ret)
+ goto err_unregister_nb;
+
+ container = group->container;
+ down_read(&container->group_lock);
+
+ driver = container->iommu_driver;
+ if (likely(driver && driver->ops->unregister_notifier))
+ ret = driver->ops->unregister_notifier(container->iommu_data,
+ nb);
+ else
+ ret = -ENOTTY;
+
+ up_read(&container->group_lock);
+ vfio_group_try_dissolve_container(group);
+
+err_unregister_nb:
+ vfio_group_put(group);
+ return ret;
+}
+EXPORT_SYMBOL(vfio_unregister_notifier);
+
/**
* Module/class support
*/
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 2697d874dd35..7e034e7c5ea5 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -38,6 +38,7 @@
#include <linux/workqueue.h>
#include <linux/pid_namespace.h>
#include <linux/mdev.h>
+#include <linux/notifier.h>
#define DRIVER_VERSION "0.2"
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
@@ -60,6 +61,7 @@ struct vfio_iommu {
struct vfio_domain *external_domain; /* domain for external user */
struct mutex lock;
struct rb_root dma_list;
+ struct blocking_notifier_head notifier;
bool v2;
bool nesting;
};
@@ -568,7 +570,8 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
mutex_lock(&iommu->lock);
- if (!iommu->external_domain) {
+ /* Fail if notifier list is empty */
+ if ((!iommu->external_domain) || (!iommu->notifier.head)) {
ret = -EINVAL;
goto pin_done;
}
@@ -854,15 +857,29 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
if (dma->task->mm != current->mm)
break;
unmapped += dma->size;
+
+ mutex_unlock(&iommu->lock);
+ if (iommu->external_domain && !RB_EMPTY_ROOT(&dma->pfn_list)) {
+ struct vfio_iommu_type1_dma_unmap nb_unmap;
+
+ nb_unmap.iova = dma->iova;
+ nb_unmap.size = dma->size;
+ blocking_notifier_call_chain(&iommu->notifier,
+ VFIO_IOMMU_NOTIFY_DMA_UNMAP,
+ &nb_unmap);
+
+ if (WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)))
+ goto unmap_exit;
+ }
+ mutex_lock(&iommu->lock);
vfio_remove_dma(iommu, dma);
}
unlock:
mutex_unlock(&iommu->lock);
-
+unmap_exit:
/* Report how much was unmapped */
unmap->size = unmapped;
-
return ret;
}
@@ -1430,6 +1447,7 @@ static void *vfio_iommu_type1_open(unsigned long arg)
INIT_LIST_HEAD(&iommu->domain_list);
iommu->dma_list = RB_ROOT;
mutex_init(&iommu->lock);
+ BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier);
return iommu;
}
@@ -1565,16 +1583,34 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
return -ENOTTY;
}
+static int vfio_iommu_type1_register_notifier(void *iommu_data,
+ struct notifier_block *nb)
+{
+ struct vfio_iommu *iommu = iommu_data;
+
+ return blocking_notifier_chain_register(&iommu->notifier, nb);
+}
+
+static int vfio_iommu_type1_unregister_notifier(void *iommu_data,
+ struct notifier_block *nb)
+{
+ struct vfio_iommu *iommu = iommu_data;
+
+ return blocking_notifier_chain_unregister(&iommu->notifier, nb);
+}
+
static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
- .name = "vfio-iommu-type1",
- .owner = THIS_MODULE,
- .open = vfio_iommu_type1_open,
- .release = vfio_iommu_type1_release,
- .ioctl = vfio_iommu_type1_ioctl,
- .attach_group = vfio_iommu_type1_attach_group,
- .detach_group = vfio_iommu_type1_detach_group,
- .pin_pages = vfio_iommu_type1_pin_pages,
- .unpin_pages = vfio_iommu_type1_unpin_pages,
+ .name = "vfio-iommu-type1",
+ .owner = THIS_MODULE,
+ .open = vfio_iommu_type1_open,
+ .release = vfio_iommu_type1_release,
+ .ioctl = vfio_iommu_type1_ioctl,
+ .attach_group = vfio_iommu_type1_attach_group,
+ .detach_group = vfio_iommu_type1_detach_group,
+ .pin_pages = vfio_iommu_type1_pin_pages,
+ .unpin_pages = vfio_iommu_type1_unpin_pages,
+ .register_notifier = vfio_iommu_type1_register_notifier,
+ .unregister_notifier = vfio_iommu_type1_unregister_notifier,
};
static int __init vfio_iommu_type1_init(void)
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 86f507d0f585..123e089388e9 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -80,6 +80,10 @@ struct vfio_iommu_driver_ops {
unsigned long *phys_pfn);
int (*unpin_pages)(void *iommu_data,
unsigned long *user_pfn, int npage);
+ int (*register_notifier)(void *iommu_data,
+ struct notifier_block *nb);
+ int (*unregister_notifier)(void *iommu_data,
+ struct notifier_block *nb);
};
extern int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops);
@@ -137,6 +141,13 @@ extern int vfio_pin_pages(struct device *dev, unsigned long *user_pfn,
extern int vfio_unpin_pages(struct device *dev, unsigned long *user_pfn,
int npage);
+#define VFIO_IOMMU_NOTIFY_DMA_UNMAP 1
+
+extern int vfio_register_notifier(struct device *dev,
+ struct notifier_block *nb);
+
+extern int vfio_unregister_notifier(struct device *dev,
+ struct notifier_block *nb);
/*
* IRQfd - generic
*/
--
2.7.0
^ permalink raw reply related
* [Qemu-devel] [PATCH v12 09/22] vfio iommu type1: Add task structure to vfio_dma
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Add task structure to vfio_dma structure.
During DMA_UNMAP, same task who mapped it or other task who shares same
address space is allowed to unmap, otherwise unmap fails.
QEMU maps few iova ranges initially, then fork threads and from the child
thread calls DMA_UNMAP on previously mapped iova. Since child shares same
address space, DMA_UNMAP is successful.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I7600f1bea6b384fd589fa72421ccf031bcfd9ac5
---
drivers/vfio/vfio_iommu_type1.c | 137 +++++++++++++++++++++++++---------------
1 file changed, 86 insertions(+), 51 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index ffe2026f1341..50aca95cf61e 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -36,6 +36,7 @@
#include <linux/uaccess.h>
#include <linux/vfio.h>
#include <linux/workqueue.h>
+#include <linux/pid_namespace.h>
#define DRIVER_VERSION "0.2"
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
@@ -75,6 +76,7 @@ struct vfio_dma {
unsigned long vaddr; /* Process virtual addr */
size_t size; /* Map size (bytes) */
int prot; /* IOMMU_READ/WRITE */
+ struct task_struct *task;
};
struct vfio_group {
@@ -277,41 +279,47 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
* the iommu can only map chunks of consecutive pfns anyway, so get the
* first page and all consecutive pages with the same locking.
*/
-static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
- int prot, unsigned long *pfn_base)
+static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
+ long npage, int prot, unsigned long *pfn_base)
{
- unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
- bool lock_cap = capable(CAP_IPC_LOCK);
+ unsigned long limit;
+ bool lock_cap = ns_capable(task_active_pid_ns(dma->task)->user_ns,
+ CAP_IPC_LOCK);
+ struct mm_struct *mm;
long ret, i;
bool rsvd;
- if (!current->mm)
+ mm = get_task_mm(dma->task);
+ if (!mm)
return -ENODEV;
- ret = vaddr_get_pfn(current->mm, vaddr, prot, pfn_base);
+ ret = vaddr_get_pfn(mm, vaddr, prot, pfn_base);
if (ret)
- return ret;
+ goto pin_pg_remote_exit;
rsvd = is_invalid_reserved_pfn(*pfn_base);
+ limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
- if (!rsvd && !lock_cap && current->mm->locked_vm + 1 > limit) {
+ if (!rsvd && !lock_cap && mm->locked_vm + 1 > limit) {
put_pfn(*pfn_base, prot);
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
limit << PAGE_SHIFT);
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto pin_pg_remote_exit;
}
if (unlikely(disable_hugepages)) {
if (!rsvd)
- vfio_lock_acct(current, 1);
- return 1;
+ vfio_lock_acct(dma->task, 1);
+ ret = 1;
+ goto pin_pg_remote_exit;
}
/* Lock all the consecutive pages from pfn_base */
for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
unsigned long pfn = 0;
- ret = vaddr_get_pfn(current->mm, vaddr, prot, &pfn);
+ ret = vaddr_get_pfn(mm, vaddr, prot, &pfn);
if (ret)
break;
@@ -321,8 +329,7 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
break;
}
- if (!rsvd && !lock_cap &&
- current->mm->locked_vm + i + 1 > limit) {
+ if (!rsvd && !lock_cap && mm->locked_vm + i + 1 > limit) {
put_pfn(pfn, prot);
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
__func__, limit << PAGE_SHIFT);
@@ -331,13 +338,16 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
}
if (!rsvd)
- vfio_lock_acct(current, i);
+ vfio_lock_acct(dma->task, i);
+ ret = i;
- return i;
+pin_pg_remote_exit:
+ mmput(mm);
+ return ret;
}
-static long vfio_unpin_pages_remote(unsigned long pfn, long npage,
- int prot, bool do_accounting)
+static long vfio_unpin_pages_remote(struct vfio_dma *dma, unsigned long pfn,
+ long npage, int prot, bool do_accounting)
{
unsigned long unlocked = 0;
long i;
@@ -346,7 +356,7 @@ static long vfio_unpin_pages_remote(unsigned long pfn, long npage,
unlocked += put_pfn(pfn++, prot);
if (do_accounting)
- vfio_lock_acct(current, -unlocked);
+ vfio_lock_acct(dma->task, -unlocked);
return unlocked;
}
@@ -400,7 +410,7 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
if (WARN_ON(!unmapped))
break;
- unlocked += vfio_unpin_pages_remote(phys >> PAGE_SHIFT,
+ unlocked += vfio_unpin_pages_remote(dma, phys >> PAGE_SHIFT,
unmapped >> PAGE_SHIFT,
dma->prot, false);
iova += unmapped;
@@ -408,13 +418,14 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
cond_resched();
}
- vfio_lock_acct(current, -unlocked);
+ vfio_lock_acct(dma->task, -unlocked);
}
static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
{
vfio_unmap_unpin(iommu, dma);
vfio_unlink_dma(iommu, dma);
+ put_task_struct(dma->task);
kfree(dma);
}
@@ -510,6 +521,12 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) {
if (!iommu->v2 && unmap->iova > dma->iova)
break;
+ /*
+ * Task with same address space who mapped this iova range is
+ * allowed to unmap the iova range.
+ */
+ if (dma->task->mm != current->mm)
+ break;
unmapped += dma->size;
vfio_remove_dma(iommu, dma);
}
@@ -576,17 +593,55 @@ unwind:
return ret;
}
+static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
+ size_t map_size)
+{
+ dma_addr_t iova = dma->iova;
+ unsigned long vaddr = dma->vaddr;
+ size_t size = map_size;
+ long npage;
+ unsigned long pfn;
+ int ret = 0;
+
+ while (size) {
+ /* Pin a contiguous chunk of memory */
+ npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
+ size >> PAGE_SHIFT, dma->prot,
+ &pfn);
+ if (npage <= 0) {
+ WARN_ON(!npage);
+ ret = (int)npage;
+ break;
+ }
+
+ /* Map it! */
+ ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,
+ dma->prot);
+ if (ret) {
+ vfio_unpin_pages_remote(dma, pfn, npage,
+ dma->prot, true);
+ break;
+ }
+
+ size -= npage << PAGE_SHIFT;
+ dma->size += npage << PAGE_SHIFT;
+ }
+
+ if (ret)
+ vfio_remove_dma(iommu, dma);
+
+ return ret;
+}
+
static int vfio_dma_do_map(struct vfio_iommu *iommu,
struct vfio_iommu_type1_dma_map *map)
{
dma_addr_t iova = map->iova;
unsigned long vaddr = map->vaddr;
size_t size = map->size;
- long npage;
int ret = 0, prot = 0;
uint64_t mask;
struct vfio_dma *dma;
- unsigned long pfn;
/* Verify that none of our __u64 fields overflow */
if (map->size != size || map->vaddr != vaddr || map->iova != iova)
@@ -612,47 +667,27 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
mutex_lock(&iommu->lock);
if (vfio_find_dma(iommu, iova, size)) {
- mutex_unlock(&iommu->lock);
- return -EEXIST;
+ ret = -EEXIST;
+ goto do_map_err;
}
dma = kzalloc(sizeof(*dma), GFP_KERNEL);
if (!dma) {
- mutex_unlock(&iommu->lock);
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto do_map_err;
}
dma->iova = iova;
dma->vaddr = vaddr;
dma->prot = prot;
+ get_task_struct(current);
+ dma->task = current;
/* Insert zero-sized and grow as we map chunks of it */
vfio_link_dma(iommu, dma);
- while (size) {
- /* Pin a contiguous chunk of memory */
- npage = vfio_pin_pages_remote(vaddr + dma->size,
- size >> PAGE_SHIFT, prot, &pfn);
- if (npage <= 0) {
- WARN_ON(!npage);
- ret = (int)npage;
- break;
- }
-
- /* Map it! */
- ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage, prot);
- if (ret) {
- vfio_unpin_pages_remote(pfn, npage, prot, true);
- break;
- }
-
- size -= npage << PAGE_SHIFT;
- dma->size += npage << PAGE_SHIFT;
- }
-
- if (ret)
- vfio_remove_dma(iommu, dma);
-
+ ret = vfio_pin_map_dma(iommu, dma, size);
+do_map_err:
mutex_unlock(&iommu->lock);
return ret;
}
--
2.7.0
^ permalink raw reply related
* [Qemu-devel] [PATCH v12 08/22] vfio iommu type1: Add find_iommu_group() function
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Add find_iommu_group()
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Reviewed-by: Jike Song <jike.song@intel.com>
Reviewed-by: Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
Change-Id: I9d372f1ebe9eb01a5a21374b8a2b03f7df73601f
---
drivers/vfio/vfio_iommu_type1.c | 57 ++++++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 24 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 52af5fc01d91..ffe2026f1341 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -752,11 +752,24 @@ static void vfio_test_domain_fgsp(struct vfio_domain *domain)
__free_pages(pages, order);
}
+static struct vfio_group *find_iommu_group(struct vfio_domain *domain,
+ struct iommu_group *iommu_group)
+{
+ struct vfio_group *g;
+
+ list_for_each_entry(g, &domain->group_list, next) {
+ if (g->iommu_group == iommu_group)
+ return g;
+ }
+
+ return NULL;
+}
+
static int vfio_iommu_type1_attach_group(void *iommu_data,
struct iommu_group *iommu_group)
{
struct vfio_iommu *iommu = iommu_data;
- struct vfio_group *group, *g;
+ struct vfio_group *group;
struct vfio_domain *domain, *d;
struct bus_type *bus = NULL;
int ret;
@@ -764,10 +777,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
mutex_lock(&iommu->lock);
list_for_each_entry(d, &iommu->domain_list, next) {
- list_for_each_entry(g, &d->group_list, next) {
- if (g->iommu_group != iommu_group)
- continue;
-
+ if (find_iommu_group(d, iommu_group)) {
mutex_unlock(&iommu->lock);
return -EINVAL;
}
@@ -887,27 +897,26 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
mutex_lock(&iommu->lock);
list_for_each_entry(domain, &iommu->domain_list, next) {
- list_for_each_entry(group, &domain->group_list, next) {
- if (group->iommu_group != iommu_group)
- continue;
+ group = find_iommu_group(domain, iommu_group);
+ if (!group)
+ continue;
- iommu_detach_group(domain->domain, iommu_group);
- list_del(&group->next);
- kfree(group);
- /*
- * Group ownership provides privilege, if the group
- * list is empty, the domain goes away. If it's the
- * last domain, then all the mappings go away too.
- */
- if (list_empty(&domain->group_list)) {
- if (list_is_singular(&iommu->domain_list))
- vfio_iommu_unmap_unpin_all(iommu);
- iommu_domain_free(domain->domain);
- list_del(&domain->next);
- kfree(domain);
- }
- goto done;
+ iommu_detach_group(domain->domain, iommu_group);
+ list_del(&group->next);
+ kfree(group);
+ /*
+ * Group ownership provides privilege, if the group
+ * list is empty, the domain goes away. If it's the
+ * last domain, then all the mappings go away too.
+ */
+ if (list_empty(&domain->group_list)) {
+ if (list_is_singular(&iommu->domain_list))
+ vfio_iommu_unmap_unpin_all(iommu);
+ iommu_domain_free(domain->domain);
+ list_del(&domain->next);
+ kfree(domain);
}
+ goto done;
}
done:
--
2.7.0
^ permalink raw reply related
* Re: [PATCH v2] ARM: at91/dt: add dts file for sama5d36ek CMP board
From: Sudeep Holla @ 2016-11-14 15:41 UTC (permalink / raw)
To: Wenyou Yang
Cc: Mark Rutland, devicetree, Russell King, Wenyou Yang, Pawel Moll,
Ian Campbell, Sudeep Holla, Nicolas Ferre, linux-kernel,
Rob Herring, Alexandre Belloni, Kumar Gala, linux-arm-kernel
In-Reply-To: <1478055958-8463-1-git-send-email-wenyou.yang@atmel.com>
On 02/11/16 03:05, Wenyou Yang wrote:
> The sama5d36ek CMP board is the variant of sama5d3xek board.
> It is equipped with the low-power DDR2 SDRAM, PMIC ACT8865 and
> some power rail. Its main purpose is used to measure the power
> consumption.
> The difference of the sama5d36ek CMP dts from sama5d36ek dts
> is listed as below.
> 1. The USB host nodes are removed, that is, the USB host is disabled.
> 2. The gpio_keys node is added to wake up from the sleep.
> 3. The LCD isn't supported due to the pins for LCD are conflicted
> with gpio_keys.
> 4. The adc0 node support the pinctrl sleep state to fix the over
> consumption on VDDANA.
>
> As said in errata, "When the USB host ports are used in high speed
> mode (EHCI), it is not possible to suspend the ports if no device is
> attached on each port. This leads to increased power consumption even
> if the system is in a low power mode." That is why the the USB host
> is disabled.
>
> Signed-off-by: Wenyou Yang <wenyou.yang@atmel.com>
> ---
>
> Changes in v2:
> - Add the pinctrl sleep state for adc0 node to fix the over
> consumption on VDDANA.
> - Improve the commit log.
>
> arch/arm/boot/dts/sama5d36ek_cmp.dts | 51 +++++++
> arch/arm/boot/dts/sama5d3xcm_cmp.dtsi | 166 +++++++++++++++++++++
> arch/arm/boot/dts/sama5d3xmb_cmp.dtsi | 265 ++++++++++++++++++++++++++++++++++
> 3 files changed, 482 insertions(+)
> create mode 100644 arch/arm/boot/dts/sama5d36ek_cmp.dts
> create mode 100644 arch/arm/boot/dts/sama5d3xcm_cmp.dtsi
> create mode 100644 arch/arm/boot/dts/sama5d3xmb_cmp.dtsi
>
[...]
> + /* Conflict with LCD pins */
> + gpio_keys {
> + compatible = "gpio-keys";
> + status = "okay";
> +
> + #address-cells = <1>;
> + #size-cells = <0>;
> + pinctrl-names = "default";
> + pinctrl-0 = <&pinctrl_gpio_keys>;
> +
> + pb_user1 {
> + label = "pb_user1";
> + gpios = <&pioE 27 GPIO_ACTIVE_HIGH>;
> + linux,code = <0x100>;
> + gpio-key,wakeup;
Please replace this with "wakeup-source", I will post patches to fix the
new comers in the mainline. It was cleaned last year.
--
Regards,
Sudeep
^ permalink raw reply
* [PATCH v2] ARM: at91/dt: add dts file for sama5d36ek CMP board
From: Sudeep Holla @ 2016-11-14 15:41 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1478055958-8463-1-git-send-email-wenyou.yang@atmel.com>
On 02/11/16 03:05, Wenyou Yang wrote:
> The sama5d36ek CMP board is the variant of sama5d3xek board.
> It is equipped with the low-power DDR2 SDRAM, PMIC ACT8865 and
> some power rail. Its main purpose is used to measure the power
> consumption.
> The difference of the sama5d36ek CMP dts from sama5d36ek dts
> is listed as below.
> 1. The USB host nodes are removed, that is, the USB host is disabled.
> 2. The gpio_keys node is added to wake up from the sleep.
> 3. The LCD isn't supported due to the pins for LCD are conflicted
> with gpio_keys.
> 4. The adc0 node support the pinctrl sleep state to fix the over
> consumption on VDDANA.
>
> As said in errata, "When the USB host ports are used in high speed
> mode (EHCI), it is not possible to suspend the ports if no device is
> attached on each port. This leads to increased power consumption even
> if the system is in a low power mode." That is why the the USB host
> is disabled.
>
> Signed-off-by: Wenyou Yang <wenyou.yang@atmel.com>
> ---
>
> Changes in v2:
> - Add the pinctrl sleep state for adc0 node to fix the over
> consumption on VDDANA.
> - Improve the commit log.
>
> arch/arm/boot/dts/sama5d36ek_cmp.dts | 51 +++++++
> arch/arm/boot/dts/sama5d3xcm_cmp.dtsi | 166 +++++++++++++++++++++
> arch/arm/boot/dts/sama5d3xmb_cmp.dtsi | 265 ++++++++++++++++++++++++++++++++++
> 3 files changed, 482 insertions(+)
> create mode 100644 arch/arm/boot/dts/sama5d36ek_cmp.dts
> create mode 100644 arch/arm/boot/dts/sama5d3xcm_cmp.dtsi
> create mode 100644 arch/arm/boot/dts/sama5d3xmb_cmp.dtsi
>
[...]
> + /* Conflict with LCD pins */
> + gpio_keys {
> + compatible = "gpio-keys";
> + status = "okay";
> +
> + #address-cells = <1>;
> + #size-cells = <0>;
> + pinctrl-names = "default";
> + pinctrl-0 = <&pinctrl_gpio_keys>;
> +
> + pb_user1 {
> + label = "pb_user1";
> + gpios = <&pioE 27 GPIO_ACTIVE_HIGH>;
> + linux,code = <0x100>;
> + gpio-key,wakeup;
Please replace this with "wakeup-source", I will post patches to fix the
new comers in the mainline. It was cleaned last year.
--
Regards,
Sudeep
^ permalink raw reply
* [Qemu-devel] [PATCH v12 07/22] vfio iommu type1: Update argument of vaddr_get_pfn()
From: Kirti Wankhede @ 2016-11-14 15:42 UTC (permalink / raw)
To: alex.williamson, pbonzini, kraxel, cjia
Cc: qemu-devel, kvm, kevin.tian, jike.song, bjsdjshi, linux-kernel,
Kirti Wankhede
In-Reply-To: <1479138156-28905-1-git-send-email-kwankhede@nvidia.com>
Update arguments of vaddr_get_pfn() to take struct mm_struct *mm as input
argument.
Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Signed-off-by: Neo Jia <cjia@nvidia.com>
Change-Id: I885fd4cd4a9f66f4ee2c1caf58267464ec239f52
---
drivers/vfio/vfio_iommu_type1.c | 32 ++++++++++++++++++++++++--------
1 file changed, 24 insertions(+), 8 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 34d17e51dc97..52af5fc01d91 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -230,20 +230,36 @@ static int put_pfn(unsigned long pfn, int prot)
return 0;
}
-static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
+static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
+ int prot, unsigned long *pfn)
{
struct page *page[1];
struct vm_area_struct *vma;
- int ret = -EFAULT;
+ int ret;
+
+ if (mm == current->mm) {
+ ret = get_user_pages_fast(vaddr, 1, !!(prot & IOMMU_WRITE),
+ page);
+ } else {
+ unsigned int flags = 0;
+
+ if (prot & IOMMU_WRITE)
+ flags |= FOLL_WRITE;
+
+ down_read(&mm->mmap_sem);
+ ret = get_user_pages_remote(NULL, mm, vaddr, 1, flags, page,
+ NULL);
+ up_read(&mm->mmap_sem);
+ }
- if (get_user_pages_fast(vaddr, 1, !!(prot & IOMMU_WRITE), page) == 1) {
+ if (ret == 1) {
*pfn = page_to_pfn(page[0]);
return 0;
}
- down_read(¤t->mm->mmap_sem);
+ down_read(&mm->mmap_sem);
- vma = find_vma_intersection(current->mm, vaddr, vaddr + 1);
+ vma = find_vma_intersection(mm, vaddr, vaddr + 1);
if (vma && vma->vm_flags & VM_PFNMAP) {
*pfn = ((vaddr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
@@ -251,7 +267,7 @@ static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
ret = 0;
}
- up_read(¤t->mm->mmap_sem);
+ up_read(&mm->mmap_sem);
return ret;
}
@@ -272,7 +288,7 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
if (!current->mm)
return -ENODEV;
- ret = vaddr_get_pfn(vaddr, prot, pfn_base);
+ ret = vaddr_get_pfn(current->mm, vaddr, prot, pfn_base);
if (ret)
return ret;
@@ -295,7 +311,7 @@ static long vfio_pin_pages_remote(unsigned long vaddr, long npage,
for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
unsigned long pfn = 0;
- ret = vaddr_get_pfn(vaddr, prot, &pfn);
+ ret = vaddr_get_pfn(current->mm, vaddr, prot, &pfn);
if (ret)
break;
--
2.7.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.