[PATCH 0/3] virtgpu: check if P2P is possiable or not

[PATCH 0/3] virtgpu: check if P2P is possiable or not

[Julia Zhang]

To implement dGPU prime feature, virtgpu needs to import/export buffer
between virtio iGPU and passthrough dGPU. Before that, virtgpu should
check if P2P is possible or not. But calling function pci_p2pdma_distance
in guest VM will only get virtual p2pdma_distance instead of real physical
p2pdma_distance.
 
So this series introduce an implementation of virtgpu device_attach
callback to get p2pdma_distance. And also adds a new virtgpu command to
pass PCI notations from guest to host and a new xen privcmd to get physical
p2pdma_distance according to the PCI notations in host.

Julia Zhang (3):
  xen:get p2pdma_distance
  virtgpu: get p2pdma_distance
  drm/virtio: Implement device_attach

 drivers/gpu/drm/virtio/virtgpu_drv.h   | 17 ++++++++
 drivers/gpu/drm/virtio/virtgpu_prime.c | 59 +++++++++++++++++++++++++-
 drivers/gpu/drm/virtio/virtgpu_vq.c    | 53 +++++++++++++++++++++++
 drivers/xen/privcmd.c                  | 42 ++++++++++++++++++
 include/uapi/linux/virtio_gpu.h        | 19 +++++++++
 include/uapi/xen/privcmd.h             | 12 ++++++
 6 files changed, 201 insertions(+), 1 deletion(-)

-- 
2.34.1

[PATCH 1/3] xen:get p2pdma_distance

[Julia Zhang]

To get the p2pdma_distance, this create a new privcmd ioctl to calculate
p2pdma_distance for two pci devices on the host with pci notations sent
from guest virtgpu driver.

Signed-off-by: Julia Zhang <julia.zhang@amd.com>
---
 drivers/xen/privcmd.c      | 42 ++++++++++++++++++++++++++++++++++++++
 include/uapi/xen/privcmd.h | 12 +++++++++++
 2 files changed, 54 insertions(+)

diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c
index 72c161e94731..95f67815a2ef 100644
--- a/drivers/xen/privcmd.c
+++ b/drivers/xen/privcmd.c
@@ -31,6 +31,9 @@
 #include <linux/miscdevice.h>
 #include <linux/moduleparam.h>
 #include <linux/virtio_mmio.h>
+#include <linux/pci.h>
+#include <linux/pci-p2pdma.h>
+#include <linux/dma-map-ops.h>
 
 #include <asm/xen/hypervisor.h>
 #include <asm/xen/hypercall.h>
@@ -977,6 +980,42 @@ static long privcmd_ioctl_map_hva_to_gpfns(struct file *file, void __user *udata
 	return ret;
 }
 
+static int privcmd_ioctl_p2pdma_distance(struct file *file, void __user *udata)
+{
+	struct privcmd_p2pdma_distance kdata;
+	struct pci_dev *provider = NULL;
+	struct pci_dev *client = NULL;
+	struct pci_dev *dev = NULL;
+	enum pci_p2pdma_map_type map;
+
+	if (copy_from_user(&kdata, udata, sizeof(kdata)))
+		return -EFAULT;
+
+	for_each_pci_dev(dev) {
+		if (dev->bus->number == kdata.provider_bus &&
+		    dev->devfn == PCI_DEVFN(kdata.provider_slot, kdata.provider_func)) {
+			provider = dev;
+		} else if (dev->bus->number == kdata.client_bus &&
+			   dev->devfn == PCI_DEVFN(kdata.client_slot, kdata.client_func)) {
+			client = dev;
+		} else {
+			continue;
+		}
+	}
+
+	if (!provider || !client) {
+		pr_err("%s fail to get provider or client.\n", __func__);
+		return -EINVAL;
+	}
+
+	kdata.distance = pci_p2pdma_distance(provider, &client->dev, false);
+
+	if (copy_to_user(udata, &kdata, sizeof(kdata)))
+		return -EFAULT;
+
+	return 0;
+}
+
 #ifdef CONFIG_XEN_PRIVCMD_EVENTFD
 /* Irqfd support */
 static struct workqueue_struct *irqfd_cleanup_wq;
@@ -1684,6 +1723,9 @@ static long privcmd_ioctl(struct file *file,
 		ret = privcmd_ioctl_map_hva_to_gpfns(file, udata);
 		break;
 
+	case IOCTL_PRIVCMD_P2PDMA_DISTANCE:
+		ret = privcmd_ioctl_p2pdma_distance(file, udata);
+		break;
 
 	default:
 		break;
diff --git a/include/uapi/xen/privcmd.h b/include/uapi/xen/privcmd.h
index d131002dd48f..a7ec3704519f 100644
--- a/include/uapi/xen/privcmd.h
+++ b/include/uapi/xen/privcmd.h
@@ -141,6 +141,16 @@ struct privcmd_map_hva_to_gpfns {
 	int add_mapping;
 };
 
+struct privcmd_p2pdma_distance {
+	__u32 provider_bus;
+	__u32 provider_slot;
+	__u32 provider_func;
+	__u32 client_bus;
+	__u32 client_slot;
+	__u32 client_func;
+	__u32 distance;
+};
+
 /*
  * @cmd: IOCTL_PRIVCMD_HYPERCALL
  * @arg: &privcmd_hypercall_t
@@ -174,6 +184,8 @@ struct privcmd_map_hva_to_gpfns {
 	_IOW('P', 9, struct privcmd_ioeventfd)
 #define IOCTL_PRIVCMD_PCIDEV_GET_GSI				\
 	_IOC(_IOC_NONE, 'P', 10, sizeof(struct privcmd_pcidev_get_gsi))
+#define IOCTL_PRIVCMD_P2PDMA_DISTANCE                          \
+	_IOC(_IOC_NONE, 'P', 11, sizeof(struct privcmd_p2pdma_distance))
 #define IOCTL_PRIVCMD_MAP_HVA_TO_GPFNS                          \
 	_IOC(_IOC_NONE, 'P', 13, sizeof(struct privcmd_map_hva_to_gpfns))
 
-- 
2.34.1

[PATCH 2/3] virtgpu: get p2pdma_distance

[Julia Zhang]

virtgpu driver need to get the p2pdma_distance to decide if virtio
iGPU buffer can be imported by passthrough dGPU, which is the main
step implement dGPU prime feature.

To get the p2pdma_distance, this create a new virtgpu ioctl to send
command of getting p2pdma_distance and pci notations of two pci devices
from guest to host. Host will handle ioctl to calculate p2pdma_distance
for this two pci devices on the host with pci notations sent from guest
virtgpu driver.

Signed-off-by: Julia Zhang <julia.zhang@amd.com>
---
 drivers/gpu/drm/virtio/virtgpu_drv.h   | 17 +++++++++
 drivers/gpu/drm/virtio/virtgpu_prime.c | 45 ++++++++++++++++++++++
 drivers/gpu/drm/virtio/virtgpu_vq.c    | 53 ++++++++++++++++++++++++++
 include/uapi/linux/virtio_gpu.h        | 19 +++++++++
 4 files changed, 134 insertions(+)

diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index 64c236169db8..1bbe510c8410 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -87,6 +87,18 @@ struct virtio_gpu_object_params {
 	uint64_t blob_id;
 };
 
+struct virtio_gpu_pci_notation {
+        uint32_t bus;
+        uint32_t slot;
+        uint32_t func;
+};
+
+struct virtio_gpu_p2pdma_distance {
+        atomic_t p2pdma_state;
+        uint32_t distance;
+};
+
+
 struct virtio_gpu_object {
 	struct drm_gem_shmem_object base;
 	uint32_t hw_res_handle;
@@ -488,4 +500,9 @@ void virtio_gpu_vram_unmap_dma_buf(struct device *dev,
 int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file);
 
+int virtio_gpu_cmd_p2pdma_distance(struct virtio_gpu_device *vgdev,
+                                   struct virtio_gpu_pci_notation *provider_params,
+                                   struct virtio_gpu_pci_notation *client_params,
+                                   struct virtio_gpu_p2pdma_distance *p2pdma_dist);
+
 #endif
diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c
index 44425f20d91a..4960620eba02 100644
--- a/drivers/gpu/drm/virtio/virtgpu_prime.c
+++ b/drivers/gpu/drm/virtio/virtgpu_prime.c
@@ -23,6 +23,8 @@
  */
 
 #include <drm/drm_prime.h>
+#include <linux/pci.h>
+#include <linux/pci-p2pdma.h>
 #include <linux/virtio_dma_buf.h>
 
 #include "virtgpu_drv.h"
@@ -71,6 +73,49 @@ static void virtgpu_gem_unmap_dma_buf(struct dma_buf_attachment *attach,
 	drm_gem_unmap_dma_buf(attach, sgt, dir);
 }
 
+static int virtgpu_get_p2pdma_distance(struct dma_buf *dma_buf,
+				       struct dma_buf_attachment *attach)
+{
+	struct drm_gem_object *obj = attach->dmabuf->priv;
+	struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(obj);
+	struct drm_device *dev = obj->dev;
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	struct pci_dev *pci_client;
+	struct virtio_gpu_pci_notation provider_params = {0};
+	struct virtio_gpu_pci_notation client_params = {0};
+	struct virtio_gpu_device *vgdev = dev->dev_private;
+	struct virtio_gpu_p2pdma_distance p2pdma_dist = {0};
+	int ret = 0;
+
+	attach->peer2peer = false;
+	if (pci_p2pdma_distance(pdev, attach->dev, false) < 0)
+		return 0;
+
+	provider_params.bus = pdev->bus->number;
+	provider_params.slot = PCI_SLOT(pdev->devfn);
+	provider_params.func = PCI_FUNC(pdev->devfn);
+	pci_client = to_pci_dev(attach->dev);
+	client_params.bus = pci_client->bus->number;
+	client_params.slot = PCI_SLOT(pci_client->devfn);
+	client_params.func = PCI_FUNC(pci_client->devfn);
+	ret = virtio_gpu_cmd_p2pdma_distance(vgdev, &provider_params, &client_params, &p2pdma_dist);
+	if (ret)
+		return ret;
+
+	ret = wait_event_timeout(vgdev->resp_wq,
+				 atomic_read(&p2pdma_dist.p2pdma_state),
+				 5 * HZ);
+	if (!ret)
+		return -EBUSY;
+
+	smp_rmb();
+
+	if (p2pdma_dist.distance >= 0)
+		attach->peer2peer = true;
+
+	return 0;
+}
+
 static const struct virtio_dma_buf_ops virtgpu_dmabuf_ops =  {
 	.ops = {
 		.cache_sgt_mapping = true,
diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
index 0d3d0d09f39b..69c33df61b88 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
@@ -1197,6 +1197,28 @@ static void virtio_gpu_cmd_resource_map_cb(struct virtio_gpu_device *vgdev,
 	wake_up_all(&vgdev->resp_wq);
 }
 
+static void virtio_gpu_cmd_p2pdma_distance_cb(struct virtio_gpu_device *vgdev,
+                                              struct virtio_gpu_vbuffer *vbuf)
+{
+	struct virtio_gpu_resp_distance *resp =
+		(struct virtio_gpu_resp_distance *)vbuf->resp_buf;
+	struct virtio_gpu_p2pdma_distance *p2pdma_dist = vbuf->resp_cb_data;
+
+	vbuf->resp_cb_data = NULL;
+
+	if (resp->hdr.type != VIRTIO_GPU_RESP_OK_P2PDMA_DISTANCE) {
+		atomic_set(&p2pdma_dist->p2pdma_state, 0);
+		goto out;
+	}
+
+	p2pdma_dist->distance = le32_to_cpu(resp->distance);
+	smp_wmb();
+	atomic_set(&p2pdma_dist->p2pdma_state, 1);
+
+out:
+	wake_up_all(&vgdev->resp_wq);
+}
+
 int virtio_gpu_cmd_map(struct virtio_gpu_device *vgdev,
 		       struct virtio_gpu_object_array *objs, uint64_t offset)
 {
@@ -1223,6 +1245,37 @@ int virtio_gpu_cmd_map(struct virtio_gpu_device *vgdev,
 	return 0;
 }
 
+int virtio_gpu_cmd_p2pdma_distance(struct virtio_gpu_device *vgdev,
+                                   struct virtio_gpu_pci_notation *provider_params,
+                                   struct virtio_gpu_pci_notation *client_params,
+                                   struct virtio_gpu_p2pdma_distance *p2pdma_dist)
+{
+	struct virtio_gpu_device_p2pdma_distance *cmd_p;
+	struct virtio_gpu_vbuffer *vbuf;
+	struct virtio_gpu_resp_distance *resp_buf;
+
+	resp_buf = kzalloc(sizeof(*resp_buf), GFP_KERNEL);
+	if (!resp_buf)
+		return -ENOMEM;
+
+	cmd_p = virtio_gpu_alloc_cmd_resp
+		(vgdev, virtio_gpu_cmd_p2pdma_distance_cb, &vbuf, sizeof(*cmd_p),
+		 sizeof(struct virtio_gpu_resp_distance), resp_buf);
+	memset(cmd_p, 0, sizeof(*cmd_p));
+
+	cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_P2PDMA_DISTANCE);
+	cmd_p->provider_bus = cpu_to_le32(provider_params->bus);
+	cmd_p->provider_slot = cpu_to_le32(provider_params->slot);
+	cmd_p->provider_func = cpu_to_le32(provider_params->func);
+	cmd_p->client_bus = cpu_to_le32(client_params->bus);
+	cmd_p->client_slot = cpu_to_le32(client_params->slot);
+	cmd_p->client_func = cpu_to_le32(client_params->func);
+	vbuf->resp_cb_data = p2pdma_dist;
+
+	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+	return 0;
+}
+
 void virtio_gpu_cmd_unmap(struct virtio_gpu_device *vgdev,
 			  struct virtio_gpu_object *bo)
 {
diff --git a/include/uapi/linux/virtio_gpu.h b/include/uapi/linux/virtio_gpu.h
index bf2c9cabd207..18313cffaa62 100644
--- a/include/uapi/linux/virtio_gpu.h
+++ b/include/uapi/linux/virtio_gpu.h
@@ -95,6 +95,7 @@ enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_CMD_SUBMIT_3D,
 	VIRTIO_GPU_CMD_RESOURCE_MAP_BLOB,
 	VIRTIO_GPU_CMD_RESOURCE_UNMAP_BLOB,
+	VIRTIO_GPU_CMD_P2PDMA_DISTANCE,
 
 	/* cursor commands */
 	VIRTIO_GPU_CMD_UPDATE_CURSOR = 0x0300,
@@ -108,6 +109,7 @@ enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_RESP_OK_EDID,
 	VIRTIO_GPU_RESP_OK_RESOURCE_UUID,
 	VIRTIO_GPU_RESP_OK_MAP_INFO,
+	VIRTIO_GPU_RESP_OK_P2PDMA_DISTANCE,
 
 	/* error responses */
 	VIRTIO_GPU_RESP_ERR_UNSPEC = 0x1200,
@@ -429,6 +431,23 @@ struct virtio_gpu_set_scanout_blob {
 	__le32 offsets[4];
 };
 
+/* VIRTIO_GPU_CMD_P2PDMA_DISTANCE */
+struct virtio_gpu_device_p2pdma_distance {
+	struct virtio_gpu_ctrl_hdr hdr;
+	__le32 provider_bus;
+	__le32 provider_slot;
+	__le32 provider_func;
+	__le32 client_bus;
+	__le32 client_slot;
+	__le32 client_func;
+};
+
+/* VIRTIO_GPU_RESP_DISTANCE */
+struct virtio_gpu_resp_distance {
+	struct virtio_gpu_ctrl_hdr hdr;
+	__le32 distance;
+};
+
 /* VIRTIO_GPU_CMD_RESOURCE_MAP_BLOB */
 struct virtio_gpu_resource_map_blob {
 	struct virtio_gpu_ctrl_hdr hdr;
-- 
2.34.1

[PATCH 3/3] drm/virtio: Implement device_attach

[Julia Zhang]

As vram objects may be imported by other gpu drivers, peer2peer flag
should be checked in dma_buf_ops->attach(). This reimplement virtio gpu
dma_buf_ops->attach by adding a device_attach() function for virtio gpu.
This function will get pci_p2pdma_distance and check attach->peer2peer
before calling drm_gem_map_attach().

Signed-off-by: Julia Zhang <julia.zhang@amd.com>
---
 drivers/gpu/drm/virtio/virtgpu_prime.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c
index 4960620eba02..4f6bce79e10e 100644
--- a/drivers/gpu/drm/virtio/virtgpu_prime.c
+++ b/drivers/gpu/drm/virtio/virtgpu_prime.c
@@ -116,6 +116,18 @@ static int virtgpu_get_p2pdma_distance(struct dma_buf *dma_buf,
 	return 0;
 }
 
+static int virtgpu_gem_device_attach(struct dma_buf *dma_buf,
+				     struct dma_buf_attachment *attach)
+{
+	int ret = virtgpu_get_p2pdma_distance(dma_buf, attach);
+	if (ret)
+		return ret;
+	if (!attach->peer2peer)
+		return -EBUSY;
+
+	return drm_gem_map_attach(dma_buf, attach);
+}
+
 static const struct virtio_dma_buf_ops virtgpu_dmabuf_ops =  {
 	.ops = {
 		.cache_sgt_mapping = true,
@@ -128,7 +140,7 @@ static const struct virtio_dma_buf_ops virtgpu_dmabuf_ops =  {
 		.vmap = drm_gem_dmabuf_vmap,
 		.vunmap = drm_gem_dmabuf_vunmap,
 	},
-	.device_attach = drm_gem_map_attach,
+	.device_attach = virtgpu_gem_device_attach,
 	.get_uuid = virtgpu_virtio_get_uuid,
 };
 
-- 
2.34.1

Re: [PATCH 3/3] drm/virtio: Implement device_attach

[Christian König]

Am 07.12.24 um 11:50 schrieb Julia Zhang:
> As vram objects may be imported by other gpu drivers, peer2peer flag
> should be checked in dma_buf_ops->attach(). This reimplement virtio gpu
> dma_buf_ops->attach by adding a device_attach() function for virtio gpu.
> This function will get pci_p2pdma_distance and check attach->peer2peer
> before calling drm_gem_map_attach().
>
> Signed-off-by: Julia Zhang <julia.zhang@amd.com>

I can't judge that virtgpu or XEN code path, but that here looks like it 
should work.

Acked-by: Christian König <christian.koenig@amd.com>

> ---
>   drivers/gpu/drm/virtio/virtgpu_prime.c | 14 +++++++++++++-
>   1 file changed, 13 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c
> index 4960620eba02..4f6bce79e10e 100644
> --- a/drivers/gpu/drm/virtio/virtgpu_prime.c
> +++ b/drivers/gpu/drm/virtio/virtgpu_prime.c
> @@ -116,6 +116,18 @@ static int virtgpu_get_p2pdma_distance(struct dma_buf *dma_buf,
>   	return 0;
>   }
>   
> +static int virtgpu_gem_device_attach(struct dma_buf *dma_buf,
> +				     struct dma_buf_attachment *attach)
> +{
> +	int ret = virtgpu_get_p2pdma_distance(dma_buf, attach);
> +	if (ret)
> +		return ret;
> +	if (!attach->peer2peer)
> +		return -EBUSY;
> +
> +	return drm_gem_map_attach(dma_buf, attach);
> +}
> +
>   static const struct virtio_dma_buf_ops virtgpu_dmabuf_ops =  {
>   	.ops = {
>   		.cache_sgt_mapping = true,
> @@ -128,7 +140,7 @@ static const struct virtio_dma_buf_ops virtgpu_dmabuf_ops =  {
>   		.vmap = drm_gem_dmabuf_vmap,
>   		.vunmap = drm_gem_dmabuf_vunmap,
>   	},
> -	.device_attach = drm_gem_map_attach,
> +	.device_attach = virtgpu_gem_device_attach,
>   	.get_uuid = virtgpu_virtio_get_uuid,
>   };
>   

Re: [PATCH 0/3] virtgpu: check if P2P is possiable or not

[Juergen Gross]

[-- Attachment #1.1.1: Type: text/plain, Size: 1062 bytes --]

On 07.12.24 11:50, Julia Zhang wrote:
> To implement dGPU prime feature, virtgpu needs to import/export buffer
> between virtio iGPU and passthrough dGPU. Before that, virtgpu should
> check if P2P is possible or not. But calling function pci_p2pdma_distance
> in guest VM will only get virtual p2pdma_distance instead of real physical
> p2pdma_distance.
>   
> So this series introduce an implementation of virtgpu device_attach
> callback to get p2pdma_distance. And also adds a new virtgpu command to
> pass PCI notations from guest to host and a new xen privcmd to get physical
> p2pdma_distance according to the PCI notations in host.

It is hard to review this series without having a clear picture how
this all is coming together.

I guess the virtgpu frontend will send a p2pdma_distance request to
the backend, which is running in user mode of dom0. This backend will
then call into the privcmd driver to obtain the needed information and
sends it back to the frontend.

Can you please confirm my suspicion is correct?


Juergen

[-- Attachment #1.1.2: OpenPGP public key --]
[-- Type: application/pgp-keys, Size: 3743 bytes --]

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

Re: [PATCH 0/3] virtgpu: check if P2P is possiable or not

[Zhang, Julia]

On 2024/12/12 15:43, Juergen Gross wrote:
> On 07.12.24 11:50, Julia Zhang wrote:
>> To implement dGPU prime feature, virtgpu needs to import/export buffer
>> between virtio iGPU and passthrough dGPU. Before that, virtgpu should
>> check if P2P is possible or not. But calling function 
>> pci_p2pdma_distance
>> in guest VM will only get virtual p2pdma_distance instead of real 
>> physical
>> p2pdma_distance.
>>   So this series introduce an implementation of virtgpu device_attach
>> callback to get p2pdma_distance. And also adds a new virtgpu command to
>> pass PCI notations from guest to host and a new xen privcmd to get 
>> physical
>> p2pdma_distance according to the PCI notations in host.
>
> It is hard to review this series without having a clear picture how
> this all is coming together.
>
> I guess the virtgpu frontend will send a p2pdma_distance request to
> the backend, which is running in user mode of dom0. This backend will
> then call into the privcmd driver to obtain the needed information and
> sends it back to the frontend.
>
> Can you please confirm my suspicion is correct?

Yes, you are right.

Julia

>
>
> Juergen