[PATCH 2/3] virtgpu: get p2pdma_distance

[Julia Zhang <julia.zhang@amd.com>]

virtgpu driver need to get the p2pdma_distance to decide if virtio
iGPU buffer can be imported by passthrough dGPU, which is the main
step implement dGPU prime feature.

To get the p2pdma_distance, this create a new virtgpu ioctl to send
command of getting p2pdma_distance and pci notations of two pci devices
from guest to host. Host will handle ioctl to calculate p2pdma_distance
for this two pci devices on the host with pci notations sent from guest
virtgpu driver.

Signed-off-by: Julia Zhang <julia.zhang@amd.com>
---
 drivers/gpu/drm/virtio/virtgpu_drv.h   | 17 +++++++++
 drivers/gpu/drm/virtio/virtgpu_prime.c | 45 ++++++++++++++++++++++
 drivers/gpu/drm/virtio/virtgpu_vq.c    | 53 ++++++++++++++++++++++++++
 include/uapi/linux/virtio_gpu.h        | 19 +++++++++
 4 files changed, 134 insertions(+)

diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index 64c236169db8..1bbe510c8410 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -87,6 +87,18 @@ struct virtio_gpu_object_params {
 	uint64_t blob_id;
 };
 
+struct virtio_gpu_pci_notation {
+        uint32_t bus;
+        uint32_t slot;
+        uint32_t func;
+};
+
+struct virtio_gpu_p2pdma_distance {
+        atomic_t p2pdma_state;
+        uint32_t distance;
+};
+
+
 struct virtio_gpu_object {
 	struct drm_gem_shmem_object base;
 	uint32_t hw_res_handle;
@@ -488,4 +500,9 @@ void virtio_gpu_vram_unmap_dma_buf(struct device *dev,
 int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file);
 
+int virtio_gpu_cmd_p2pdma_distance(struct virtio_gpu_device *vgdev,
+                                   struct virtio_gpu_pci_notation *provider_params,
+                                   struct virtio_gpu_pci_notation *client_params,
+                                   struct virtio_gpu_p2pdma_distance *p2pdma_dist);
+
 #endif
diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c
index 44425f20d91a..4960620eba02 100644
--- a/drivers/gpu/drm/virtio/virtgpu_prime.c
+++ b/drivers/gpu/drm/virtio/virtgpu_prime.c
@@ -23,6 +23,8 @@
  */
 
 #include <drm/drm_prime.h>
+#include <linux/pci.h>
+#include <linux/pci-p2pdma.h>
 #include <linux/virtio_dma_buf.h>
 
 #include "virtgpu_drv.h"
@@ -71,6 +73,49 @@ static void virtgpu_gem_unmap_dma_buf(struct dma_buf_attachment *attach,
 	drm_gem_unmap_dma_buf(attach, sgt, dir);
 }
 
+static int virtgpu_get_p2pdma_distance(struct dma_buf *dma_buf,
+				       struct dma_buf_attachment *attach)
+{
+	struct drm_gem_object *obj = attach->dmabuf->priv;
+	struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(obj);
+	struct drm_device *dev = obj->dev;
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	struct pci_dev *pci_client;
+	struct virtio_gpu_pci_notation provider_params = {0};
+	struct virtio_gpu_pci_notation client_params = {0};
+	struct virtio_gpu_device *vgdev = dev->dev_private;
+	struct virtio_gpu_p2pdma_distance p2pdma_dist = {0};
+	int ret = 0;
+
+	attach->peer2peer = false;
+	if (pci_p2pdma_distance(pdev, attach->dev, false) < 0)
+		return 0;
+
+	provider_params.bus = pdev->bus->number;
+	provider_params.slot = PCI_SLOT(pdev->devfn);
+	provider_params.func = PCI_FUNC(pdev->devfn);
+	pci_client = to_pci_dev(attach->dev);
+	client_params.bus = pci_client->bus->number;
+	client_params.slot = PCI_SLOT(pci_client->devfn);
+	client_params.func = PCI_FUNC(pci_client->devfn);
+	ret = virtio_gpu_cmd_p2pdma_distance(vgdev, &provider_params, &client_params, &p2pdma_dist);
+	if (ret)
+		return ret;
+
+	ret = wait_event_timeout(vgdev->resp_wq,
+				 atomic_read(&p2pdma_dist.p2pdma_state),
+				 5 * HZ);
+	if (!ret)
+		return -EBUSY;
+
+	smp_rmb();
+
+	if (p2pdma_dist.distance >= 0)
+		attach->peer2peer = true;
+
+	return 0;
+}
+
 static const struct virtio_dma_buf_ops virtgpu_dmabuf_ops =  {
 	.ops = {
 		.cache_sgt_mapping = true,
diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
index 0d3d0d09f39b..69c33df61b88 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
@@ -1197,6 +1197,28 @@ static void virtio_gpu_cmd_resource_map_cb(struct virtio_gpu_device *vgdev,
 	wake_up_all(&vgdev->resp_wq);
 }
 
+static void virtio_gpu_cmd_p2pdma_distance_cb(struct virtio_gpu_device *vgdev,
+                                              struct virtio_gpu_vbuffer *vbuf)
+{
+	struct virtio_gpu_resp_distance *resp =
+		(struct virtio_gpu_resp_distance *)vbuf->resp_buf;
+	struct virtio_gpu_p2pdma_distance *p2pdma_dist = vbuf->resp_cb_data;
+
+	vbuf->resp_cb_data = NULL;
+
+	if (resp->hdr.type != VIRTIO_GPU_RESP_OK_P2PDMA_DISTANCE) {
+		atomic_set(&p2pdma_dist->p2pdma_state, 0);
+		goto out;
+	}
+
+	p2pdma_dist->distance = le32_to_cpu(resp->distance);
+	smp_wmb();
+	atomic_set(&p2pdma_dist->p2pdma_state, 1);
+
+out:
+	wake_up_all(&vgdev->resp_wq);
+}
+
 int virtio_gpu_cmd_map(struct virtio_gpu_device *vgdev,
 		       struct virtio_gpu_object_array *objs, uint64_t offset)
 {
@@ -1223,6 +1245,37 @@ int virtio_gpu_cmd_map(struct virtio_gpu_device *vgdev,
 	return 0;
 }
 
+int virtio_gpu_cmd_p2pdma_distance(struct virtio_gpu_device *vgdev,
+                                   struct virtio_gpu_pci_notation *provider_params,
+                                   struct virtio_gpu_pci_notation *client_params,
+                                   struct virtio_gpu_p2pdma_distance *p2pdma_dist)
+{
+	struct virtio_gpu_device_p2pdma_distance *cmd_p;
+	struct virtio_gpu_vbuffer *vbuf;
+	struct virtio_gpu_resp_distance *resp_buf;
+
+	resp_buf = kzalloc(sizeof(*resp_buf), GFP_KERNEL);
+	if (!resp_buf)
+		return -ENOMEM;
+
+	cmd_p = virtio_gpu_alloc_cmd_resp
+		(vgdev, virtio_gpu_cmd_p2pdma_distance_cb, &vbuf, sizeof(*cmd_p),
+		 sizeof(struct virtio_gpu_resp_distance), resp_buf);
+	memset(cmd_p, 0, sizeof(*cmd_p));
+
+	cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_P2PDMA_DISTANCE);
+	cmd_p->provider_bus = cpu_to_le32(provider_params->bus);
+	cmd_p->provider_slot = cpu_to_le32(provider_params->slot);
+	cmd_p->provider_func = cpu_to_le32(provider_params->func);
+	cmd_p->client_bus = cpu_to_le32(client_params->bus);
+	cmd_p->client_slot = cpu_to_le32(client_params->slot);
+	cmd_p->client_func = cpu_to_le32(client_params->func);
+	vbuf->resp_cb_data = p2pdma_dist;
+
+	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+	return 0;
+}
+
 void virtio_gpu_cmd_unmap(struct virtio_gpu_device *vgdev,
 			  struct virtio_gpu_object *bo)
 {
diff --git a/include/uapi/linux/virtio_gpu.h b/include/uapi/linux/virtio_gpu.h
index bf2c9cabd207..18313cffaa62 100644
--- a/include/uapi/linux/virtio_gpu.h
+++ b/include/uapi/linux/virtio_gpu.h
@@ -95,6 +95,7 @@ enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_CMD_SUBMIT_3D,
 	VIRTIO_GPU_CMD_RESOURCE_MAP_BLOB,
 	VIRTIO_GPU_CMD_RESOURCE_UNMAP_BLOB,
+	VIRTIO_GPU_CMD_P2PDMA_DISTANCE,
 
 	/* cursor commands */
 	VIRTIO_GPU_CMD_UPDATE_CURSOR = 0x0300,
@@ -108,6 +109,7 @@ enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_RESP_OK_EDID,
 	VIRTIO_GPU_RESP_OK_RESOURCE_UUID,
 	VIRTIO_GPU_RESP_OK_MAP_INFO,
+	VIRTIO_GPU_RESP_OK_P2PDMA_DISTANCE,
 
 	/* error responses */
 	VIRTIO_GPU_RESP_ERR_UNSPEC = 0x1200,
@@ -429,6 +431,23 @@ struct virtio_gpu_set_scanout_blob {
 	__le32 offsets[4];
 };
 
+/* VIRTIO_GPU_CMD_P2PDMA_DISTANCE */
+struct virtio_gpu_device_p2pdma_distance {
+	struct virtio_gpu_ctrl_hdr hdr;
+	__le32 provider_bus;
+	__le32 provider_slot;
+	__le32 provider_func;
+	__le32 client_bus;
+	__le32 client_slot;
+	__le32 client_func;
+};
+
+/* VIRTIO_GPU_RESP_DISTANCE */
+struct virtio_gpu_resp_distance {
+	struct virtio_gpu_ctrl_hdr hdr;
+	__le32 distance;
+};
+
 /* VIRTIO_GPU_CMD_RESOURCE_MAP_BLOB */
 struct virtio_gpu_resource_map_blob {
 	struct virtio_gpu_ctrl_hdr hdr;
-- 
2.34.1