* [PATCH v4 0/3] Add support for XeLink device
@ 2023-12-20 23:17 David Kershner
2023-12-20 23:17 ` [PATCH v4 1/3] drm/xe: Introduce " David Kershner
` (2 more replies)
0 siblings, 3 replies; 4+ messages in thread
From: David Kershner @ 2023-12-20 23:17 UTC (permalink / raw)
To: david.kershner, intel-xe, michael.j.ruhl, john.fleck,
lucas.demarchi, rodrigo.vivi, matthew.d.roper, jani.nikula
First patch set introduce the needed Xe infrastructure to support
XeLink device.
The XeLink device is a glue-less module attached to a GPU device that
provides connectivity between different GPUs on the same system.
The XeLink is part of the GPU (the register space is part of the GPU
PCIe BAR), so it cannot be a completely separate device driver.
The Linux kernel provides an interface for handling this type of
device, the Auxiliary Bus API.
The Xe and XeLink will use the auxiliary bus to enable the XeLink
and the Xe to probe and communicate.
A followup series will include the XeLink driver code to provide
full functionality for the device.
Changes from version 1:
- Fix kernel-doc error
Changes from version 2:
- Fix up rebase conflicts
Changes from version 3:
- Cleaned up xe_link.h header and removed unrequired include
David Kershner (3):
drm/xe: Introduce XeLink device
drm/xe: Teach Xe how to use objects with XeLink connectivity
drm/xe/uapi: Augment query ioctl to allow for fabric
drivers/gpu/drm/xe/Makefile | 1 +
drivers/gpu/drm/xe/regs/xe_gt_regs.h | 30 ++
drivers/gpu/drm/xe/xe_bo.c | 46 ++-
drivers/gpu/drm/xe/xe_bo.h | 2 +
drivers/gpu/drm/xe/xe_device.c | 13 +-
drivers/gpu/drm/xe/xe_device_types.h | 24 ++
drivers/gpu/drm/xe/xe_dma_buf.c | 209 ++++++++----
drivers/gpu/drm/xe/xe_dma_buf.h | 3 +
drivers/gpu/drm/xe/xe_ggtt.c | 3 +
drivers/gpu/drm/xe/xe_gt_types.h | 2 +
drivers/gpu/drm/xe/xe_irq.c | 22 ++
drivers/gpu/drm/xe/xe_link.c | 478 +++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_link.h | 20 ++
drivers/gpu/drm/xe/xe_mmio.c | 3 +-
drivers/gpu/drm/xe/xe_pci.c | 2 +
drivers/gpu/drm/xe/xe_pci_types.h | 1 +
drivers/gpu/drm/xe/xe_pt.c | 2 +
drivers/gpu/drm/xe/xe_query.c | 54 +++
drivers/gpu/drm/xe/xe_trace.h | 29 ++
drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 17 +-
drivers/gpu/drm/xe/xe_vm.c | 3 +-
include/drm/xelink_platform.h | 140 ++++++++
include/uapi/drm/xe_drm.h | 26 ++
23 files changed, 1057 insertions(+), 73 deletions(-)
create mode 100644 drivers/gpu/drm/xe/xe_link.c
create mode 100644 drivers/gpu/drm/xe/xe_link.h
create mode 100644 include/drm/xelink_platform.h
--
2.38.1
^ permalink raw reply [flat|nested] 4+ messages in thread
* [PATCH v4 1/3] drm/xe: Introduce XeLink device
2023-12-20 23:17 [PATCH v4 0/3] Add support for XeLink device David Kershner
@ 2023-12-20 23:17 ` David Kershner
2023-12-20 23:17 ` [PATCH v4 2/3] drm/xe: Teach Xe how to use objects with XeLink connectivity David Kershner
2023-12-20 23:17 ` [PATCH v4 3/3] drm/xe/uapi: Augment query ioctl to allow for fabric David Kershner
2 siblings, 0 replies; 4+ messages in thread
From: David Kershner @ 2023-12-20 23:17 UTC (permalink / raw)
To: david.kershner, intel-xe, michael.j.ruhl, john.fleck,
lucas.demarchi, rodrigo.vivi, matthew.d.roper, jani.nikula
Add XeLink support for PVC devices.
Introduce the initial platform data structure that will be shared
between the Xe driver and the XeLink driver.
If the device is PVC, and there is an XeLink device present, add
and initialize an Auxbus device with the appropriate resources.
Add the XeLink information register.
Add package address register for defining the Device Physical Address
space (DPA).
Add initial interrupt support.
The package address register defines the device physical
address (DPA) space for the given device.
Using the "unique" xa_array index, generate a DPA address
for the device and program the package register appropriately.
Update the LMEM offset to use the DPA address.
Signed-off-by: Michael J. Ruhl <michael.j.ruhl@intel.com>
Signed-off-by: David Kershner <david.kershner@intel.com>
---
drivers/gpu/drm/xe/Makefile | 1 +
drivers/gpu/drm/xe/regs/xe_gt_regs.h | 30 ++
drivers/gpu/drm/xe/xe_device.c | 13 +-
drivers/gpu/drm/xe/xe_device_types.h | 24 ++
drivers/gpu/drm/xe/xe_gt_types.h | 2 +
drivers/gpu/drm/xe/xe_irq.c | 22 ++
drivers/gpu/drm/xe/xe_link.c | 466 +++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_link.h | 18 ++
drivers/gpu/drm/xe/xe_mmio.c | 3 +-
drivers/gpu/drm/xe/xe_pci.c | 2 +
drivers/gpu/drm/xe/xe_pci_types.h | 1 +
include/drm/xelink_platform.h | 140 ++++++++
12 files changed, 719 insertions(+), 3 deletions(-)
create mode 100644 drivers/gpu/drm/xe/xe_link.c
create mode 100644 drivers/gpu/drm/xe/xe_link.h
create mode 100644 include/drm/xelink_platform.h
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index df8601d6a59f..c58a7e66998a 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -106,6 +106,7 @@ xe-y += xe_bb.o \
xe_huc.o \
xe_huc_debugfs.o \
xe_irq.o \
+ xe_link.o \
xe_lrc.o \
xe_migrate.o \
xe_mmio.o \
diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
index 6aaaf1f63c72..e75809669e90 100644
--- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
@@ -61,6 +61,11 @@
#define XEHP_GLOBAL_MOCS(i) XE_REG_MCR(0x4000 + (i) * 4)
#define CCS_AUX_INV XE_REG(0x4208)
+#define PKG_ADDR_RANGE XE_REG(0x41B0)
+#define PKG_ADDR_RANGE_RANGE_SHIFT 20
+#define PKG_ADDR_RANGE_BASE_SHIFT 1
+#define PKG_ADDR_RANGE_ENABLE 1
+
#define VD0_AUX_INV XE_REG(0x4218)
#define VE0_AUX_INV XE_REG(0x4238)
@@ -68,6 +73,11 @@
#define AUX_INV REG_BIT(0)
#define XEHP_TILE_ADDR_RANGE(_idx) XE_REG_MCR(0x4900 + (_idx) * 4)
+#define XEHP_TILE_LMEM_RANGE_SHIFT 8
+#define XEHP_TILE_LMEM_BASE_SHIFT 1
+#define XEHP_TILE_LMEM_BASE_MASK REG_GENMASK(7, 1)
+#define XEHP_TILE_LMEM_RANGE_MASK REG_GENMASK(14, 8)
+
#define XEHP_FLAT_CCS_BASE_ADDR XE_REG_MCR(0x4910)
#define WM_CHICKEN3 XE_REG_MCR(0x5588, XE_REG_OPTION_MASKED)
@@ -399,6 +409,11 @@
#define CCS_MODE_CSLICE(cslice, ccs) \
((ccs) << ((cslice) * CCS_MODE_CSLICE_WIDTH))
+#define PKG_ADDR_BASE XE_REG(0x108390)
+#define PKG_ADDR_BASE_RANGE_SHIFT 20
+#define PKG_ADDR_BASE_BASE_SHIFT 1
+#define PKG_ADDR_BASE_ENABLE 1
+
#define FORCEWAKE_ACK_GT XE_REG(0x130044)
#define FORCEWAKE_KERNEL BIT(0)
#define FORCEWAKE_USER BIT(1)
@@ -484,4 +499,19 @@
#define PVC_GT0_PLATFORM_ENERGY_STATUS XE_REG(0x28106c)
#define PVC_GT0_PACKAGE_POWER_SKU XE_REG(0x281080)
+#define PUNIT_MMIO_CR_POC_STRAPS XE_REG(0x281078)
+#define NUM_TILES_MASK REG_GENMASK(1, 0)
+#define CD_ALIVE REG_BIT(2)
+#define SOCKET_ID_MASK REG_GENMASK(7, 3)
+
+/* Define the BAR and offset for XeLink CSRs */
+#define XE_XELINK_IRQ BIT(8)
+#define CD_BASE_OFFSET 0x291000
+#define CD_BAR_SIZE (256 * 1024)
+
+#define CPORT_MBDB_CSRS XE_REG(CD_BASE_OFFSET + 0x6000)
+#define CPORT_MBDB_CSRS_END XE_REG(CD_BASE_OFFSET + 0x1000)
+#define CPORT_MBDB_INT_ENABLE_MASK_LOW XE_REG(CD_BASE_OFFSET + 0x8)
+#define CPORT_MBDB_INT_ENABLE_MASK_HIGH XE_REG(CD_BASE_OFFSET + 0x10)
+
#endif
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index 86867d42d532..c38e694d90ff 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: MIT
/*
- * Copyright © 2021 Intel Corporation
+ * Copyright © 2021 - 2023 Intel Corporation
*/
#include "xe_device.h"
@@ -29,6 +29,7 @@
#include "xe_gt.h"
#include "xe_gt_mcr.h"
#include "xe_irq.h"
+#include "xe_link.h"
#include "xe_memirq.h"
#include "xe_mmio.h"
#include "xe_module.h"
@@ -443,6 +444,8 @@ int xe_device_probe(struct xe_device *xe)
if (err)
return err;
+ xe_link_init_early(xe);
+
err = xe_set_dma_info(xe);
if (err)
return err;
@@ -465,6 +468,8 @@ int xe_device_probe(struct xe_device *xe)
}
}
+ xe_link_init_mmio(xe);
+
err = drmm_add_action_or_reset(&xe->drm, xe_driver_flr_fini, xe);
if (err)
return err;
@@ -493,6 +498,8 @@ int xe_device_probe(struct xe_device *xe)
if (err)
return err;
+ xe_link_init(xe);
+
err = xe_mmio_probe_vram(xe);
if (err)
goto err_irq_shutdown;
@@ -534,6 +541,8 @@ int xe_device_probe(struct xe_device *xe)
xe_display_register(xe);
+ xe_link_init_aux(xe);
+
xe_debugfs_register(xe);
xe_hwmon_register(xe);
@@ -568,6 +577,8 @@ void xe_device_remove(struct xe_device *xe)
xe_display_fini(xe);
+ xe_link_remove(xe);
+
xe_heci_gsc_fini(xe);
xe_irq_shutdown(xe);
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 71f23ac365e6..7501536cdc41 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -259,6 +259,8 @@ struct xe_device {
u8 has_asid:1;
/** @force_execlist: Forced execlist submission */
u8 force_execlist:1;
+ /** @has_xelink: Has XeLink */
+ u8 has_xelink:1;
/** @has_flat_ccs: Whether flat CCS metadata is used */
u8 has_flat_ccs:1;
/** @has_llc: Device has a shared CPU+GPU last level cache */
@@ -440,6 +442,28 @@ struct xe_device {
/** @needs_flr_on_fini: requests function-reset on fini */
bool needs_flr_on_fini;
+ /** @xelink: XeLink information, for those gpus with XeLink connectivity */
+ struct {
+ /** @ops: shared interface operations */
+ const struct xelink_ops *ops;
+ /** @handle: XeLink device handle */
+ void *handle;
+ /** @pd: platform data needed for auxiliary bus */
+ struct xelink_pdata *pd;
+ /** @dpa: base device physical address */
+ u64 dpa;
+ /** @irq_base: base IRQ for multi tile devices */
+ int irq_base;
+ /** @index: internal index for xe devices */
+ int index;
+ /** @xelink_id: XeLink id generated by the XeLink device */
+ u32 xelink_id;
+ /** @socket_id: socket from certain platforms */
+ u8 socket_id;
+ /* @present: Reflect PUNIT presence information */
+ bool present;
+ } xelink;
+
/* private: */
#if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
index f74684660475..1e8c0d3cf17f 100644
--- a/drivers/gpu/drm/xe/xe_gt_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_types.h
@@ -358,6 +358,8 @@ struct xe_gt {
/** @oob: bitmap with active OOB workaroudns */
unsigned long *oob;
} wa_active;
+ /** @xelink_irq: IRQ value assigned to the Xelink device */
+ int xelink_irq;
};
#endif
diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c
index 907c8ff0fa21..0940e8778425 100644
--- a/drivers/gpu/drm/xe/xe_irq.c
+++ b/drivers/gpu/drm/xe/xe_irq.c
@@ -5,6 +5,7 @@
#include "xe_irq.h"
+#include <linux/irq.h>
#include <linux/sched/clock.h>
#include <drm/drm_managed.h>
@@ -29,6 +30,9 @@
#define IIR(offset) XE_REG(offset + 0x8)
#define IER(offset) XE_REG(offset + 0xc)
+/* Define the BAR and offset for XeLink CSRs */
+#define XE_XELINK_IRQ BIT(8)
+
static void assert_iir_is_zero(struct xe_gt *mmio, struct xe_reg reg)
{
u32 val = xe_mmio_read32(mmio, reg);
@@ -378,6 +382,18 @@ static void dg1_intr_enable(struct xe_device *xe, bool stall)
xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR);
}
+/*
+ * xelink_irq_handler - handle XeLink IRQs
+ *
+ * PVC can have an XeLink attached. Handle the IRQs that are sourced by
+ * the device supporting the XeLink.
+ */
+static void xelink_irq_handler(struct xe_gt *gt, const u32 master_ctl)
+{
+ if (master_ctl & XE_XELINK_IRQ)
+ generic_handle_irq(gt->xelink_irq);
+}
+
/*
* Top-level interrupt handler for Xe_LP+ and beyond. These platforms have
* a "master tile" interrupt register which must be consulted before the
@@ -439,6 +455,7 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg)
xe_display_irq_handler(xe, master_ctl);
gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
}
+ xelink_irq_handler(mmio, master_ctl);
}
dg1_intr_enable(xe, false);
@@ -492,6 +509,11 @@ static void gt_irq_reset(struct xe_tile *tile)
xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_MASK, ~0);
xe_mmio_write32(mmio, GUC_SG_INTR_ENABLE, 0);
xe_mmio_write32(mmio, GUC_SG_INTR_MASK, ~0);
+
+ if (tile->xe->xelink.present) {
+ xe_mmio_write32(mmio, CPORT_MBDB_INT_ENABLE_MASK_LOW, 0);
+ xe_mmio_write32(mmio, CPORT_MBDB_INT_ENABLE_MASK_HIGH, 0);
+ }
}
static void xelp_irq_reset(struct xe_tile *tile)
diff --git a/drivers/gpu/drm/xe/xe_link.c b/drivers/gpu/drm/xe/xe_link.c
new file mode 100644
index 000000000000..f1484fbb630d
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_link.c
@@ -0,0 +1,466 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+#include <linux/auxiliary_bus.h>
+#include <linux/firmware.h>
+#include <linux/irq.h>
+#include <linux/xarray.h>
+
+#include <drm/xelink_platform.h>
+
+#include "xe_device.h"
+#include "xe_link.h"
+#include "xe_mmio.h"
+#include "xe_gt_mcr.h"
+#include "regs/xe_reg_defs.h"
+#include "regs/xe_gt_regs.h"
+
+#define HAS_XELINK(xe) ((xe)->info.has_xelink)
+/* Define the BAR and offset for the XeLink CSRs */
+#define GTTMMADR_BAR 0
+/*
+ * Define the maximum number of devices instances based on the amount of
+ * FID space.
+ *
+ * XARRAY limits are "inclusive", but using this value as a range check
+ * outside of xarray, makes the exclusive upper bound a little easier to
+ * deal with.
+ *
+ * I.e.:
+ * [0 - 256)
+ *
+ * Less than HW supports, but more than will be currently possible.
+ *
+ */
+#define MAX_DEVICE_COUNT 256
+/* Fixed Device Physical Address (DPA) size for a device/package (in GB) */
+#define MAX_DPA_SIZE 128
+
+/* Xarray of XeLink devices */
+static DEFINE_XARRAY_ALLOC(intel_fdevs);
+
+static struct query_info *default_query(void *handle, u32 xelink_id)
+{
+ return ERR_PTR(-EOPNOTSUPP);
+}
+
+static int default_handle_event(void *handle, enum xelink_parent_event event)
+{
+ return -EOPNOTSUPP;
+}
+
+static const struct xelink_ops default_ops = {
+ .connectivity_query = default_query,
+ .parent_event = default_handle_event,
+};
+
+static int register_dev(void *parent, void *handle, u32 xelink_id,
+ const struct xelink_ops *ops)
+{
+ struct xe_device *xe = parent;
+
+ WARN(xe->xelink.ops != &default_ops, "XeLink: already registered");
+
+ xe->xelink.handle = handle;
+ xe->xelink.xelink_id = xelink_id;
+ xe->xelink.ops = ops;
+
+ drm_info(&xe->drm, "XeLink: registered: 0x%x\n", xelink_id);
+
+ return 0;
+}
+
+static void unregister_dev(void *parent, const void *handle)
+{
+ struct xe_device *xe = parent;
+
+ WARN(xe->xelink.handle != handle, "XeLink: invalid handle");
+
+ drm_info(&xe->drm, "XeLink: unregistered: 0x%x\n",
+ xe->xelink.xelink_id);
+ xe->xelink.handle = NULL;
+ xe->xelink.ops = &default_ops;
+}
+
+static int dev_event(void *parent, void *handle, enum xelink_dev_event event,
+ void *event_data)
+{
+ return 0;
+}
+
+/**
+ * init_pd - Allocate and initialize platform specific data
+ * @xe: Valid xe instance
+ *
+ * Return:
+ * * pd - initialized xelink_pdata,
+ * * NULL - Allocation failure
+ */
+static struct xelink_pdata *init_pd(struct xe_device *xe)
+{
+ struct xelink_pdata *pd;
+ u32 reg;
+
+ pd = kzalloc(sizeof(*pd), GFP_KERNEL);
+ if (!pd)
+ return NULL;
+
+ pd->version = XELINK_VERSION;
+ pd->parent = xe;
+ pd->product = XELINK_PONTEVECCHIO;
+ pd->index = xe->xelink.index & 0xFFFF;
+ pd->sd_cnt = xe->info.tile_count;
+ pd->socket_id = xe->xelink.socket_id;
+ pd->slot = PCI_SLOT(to_pci_dev(xe->drm.dev)->devfn);
+
+ pd->resources = NULL;
+ pd->num_resources = 0;
+ pd->register_dev = register_dev;
+ pd->unregister_dev = unregister_dev;
+ pd->dev_event = dev_event;
+
+ /*
+ * Calculate the actual DPA offset and size (in GB) for the device.
+ * Each tile will have the same amount of memory, so we only need to
+ * read the first one.
+ */
+ reg = xe_gt_mcr_unicast_read_any(xe_device_get_root_tile(xe)->primary_gt,
+ XEHP_TILE_ADDR_RANGE(0)) & XEHP_TILE_LMEM_RANGE_MASK;
+
+ /* TILE0 is < 8Gb, PVC needs 8GB */
+ if (reg >> XEHP_TILE_LMEM_RANGE_SHIFT < 8) {
+ drm_err(&xe->drm, "XEHP_TILE0_ADDR_RANGE: %x\n", reg);
+ return NULL;
+ }
+ pd->dpa.pkg_offset = (u32)xe->xelink.index * MAX_DPA_SIZE;
+ pd->dpa.pkg_size = (reg >> XEHP_TILE_LMEM_RANGE_SHIFT) * pd->sd_cnt;
+
+ return pd;
+}
+
+/**
+ * init_resource - Create the resource array, and apply the appropriate data
+ * @xe: valid xe instance
+ * @res_cnt: pointer to return number of allocated resources
+ *
+ * First resource [0] is for the IRQ(s). Each device gets 1 IRQ. Subsequent
+ * resources describe the IO memory space for the device(s).
+ *
+ * Make sure to set the gt->xelink_irq value.
+ *
+ * Return:
+ * * res - Initialized resource array
+ * * NULL - Allocaction failure
+ */
+static struct resource *init_resource(struct xe_device *xe,
+ u32 *res_cnt)
+{
+ struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+ struct xe_gt *gt;
+ struct resource *res_base, *res;
+ u32 cnt = xe->info.tile_count * 2;
+ unsigned int i;
+
+ /* Each sd gets one resource for IRQ and one for MEM */
+ res_base = kcalloc(cnt, sizeof(*res_base), GFP_KERNEL);
+ if (!res_base)
+ return NULL;
+
+ res = res_base;
+ for_each_gt(gt, xe, i) {
+ res->start = xe->xelink.irq_base + i;
+ res->end = xe->xelink.irq_base + i;
+ res->flags = IORESOURCE_IRQ;
+ res++;
+
+ res->start = pci_resource_start(pdev, GTTMMADR_BAR) + CD_BASE_OFFSET +
+ i * gt_to_tile(gt)->mmio.size;
+ res->end = res->start + CD_BAR_SIZE - 1;
+ res->flags = IORESOURCE_MEM;
+ drm_info(&xe->drm, "XeLink: mem_resource = %pR\n", res);
+ res++;
+ gt->xelink_irq = xe->xelink.irq_base + i;
+ }
+
+ *res_cnt = cnt;
+ return res_base;
+}
+
+/**
+ * xelink_irq_mask - Null callback. Masking/unmasking happens in the parent
+ * driver
+ * @d: Valid irq_data information
+ */
+static void xelink_irq_mask(struct irq_data *d)
+{
+}
+
+static void xelink_irq_unmask(struct irq_data *d)
+{
+}
+
+static struct irq_chip xelink_irq_chip = {
+ .name = "xelink_irq_chip",
+ .irq_mask = xelink_irq_mask,
+ .irq_unmask = xelink_irq_unmask,
+};
+
+/**
+ * init_irq_desc - Allocate IRQ descriptors to use for the xelink
+ * @xe: Valid xe instance
+ *
+ * Allocate the required IRQ descriptor(s) and initialize the
+ * appropriate state.
+ *
+ * Return:
+ * * 0 - Success
+ * * errno - Error that occurred
+ */
+static int init_irq_desc(struct xe_device *xe)
+{
+ unsigned int num_subdevs = xe->info.tile_count;
+ int err;
+ int irq;
+ int irq_base;
+
+ irq_base = irq_alloc_descs(-1, 0, num_subdevs, 0);
+ if (irq_base < 0) {
+ err = irq_base;
+ goto cleanup;
+ }
+
+ err = 0;
+ for (irq = irq_base; !err && irq < irq_base + num_subdevs; irq++) {
+ irq_set_chip_and_handler_name(irq, &xelink_irq_chip,
+ handle_simple_irq,
+ "xelink_irq_handler");
+ err = irq_set_chip_data(irq, xe);
+ }
+
+ if (err) {
+ irq_free_descs(irq_base, num_subdevs);
+ goto cleanup;
+ }
+
+ drm_info(&xe->drm, "XeLink: IRQ base: %d cnt: %d\n", irq_base,
+ num_subdevs);
+
+ xe->xelink.irq_base = irq_base;
+
+ return 0;
+
+cleanup:
+ xe->xelink.index = err;
+ drm_err(&xe->drm, "XeLink: Failed to allocate IRQ data: %d\n", err);
+ return err;
+}
+
+/**
+ * xe_link_init_early - Set the XeLink info to the defaults
+ * @xe: valid xe instance
+ *
+ * index is set to ENODEV to show that, by default, there is no device.
+ * If any of the initialization steps fail, it will be set to the appropriate
+ * errno value.
+ */
+void xe_link_init_early(struct xe_device *xe)
+{
+ xe->xelink.ops = &default_ops;
+ xe->xelink.index = -ENODEV;
+}
+
+/**
+ * xe_link_init_mmio - check if XeLink is available via MMIO
+ * @xe: valid xe instance
+ *
+ * Read the relevant regs to check for XeLink availability and get the socket id
+ */
+void xe_link_init_mmio(struct xe_device *xe)
+{
+ u32 xelink_info;
+
+ if (!HAS_XELINK(xe))
+ return;
+
+ xelink_info = xe_mmio_read32(xe_device_get_root_tile(xe)->primary_gt,
+ PUNIT_MMIO_CR_POC_STRAPS);
+
+ xe->xelink.socket_id = REG_FIELD_GET(SOCKET_ID_MASK, xelink_info);
+
+ if (REG_FIELD_GET(CD_ALIVE, xelink_info)) {
+ drm_info(&xe->drm, "XeLink available\n");
+ xe->xelink.present = true;
+ }
+}
+
+/**
+ * xe_link_init - Allocate device index and complete initial HW setup
+ * @xe: valid device instance
+ *
+ * NOTE: index is zero inited. If the XeLink is not present, or an error occurs
+ * during setup, this must be 0 for the range registers.
+ *
+ */
+void xe_link_init(struct xe_device *xe)
+{
+ struct xe_gt *gt;
+ static u32 last_id;
+ unsigned int i;
+ u32 index = 0;
+ u32 range;
+ u32 base;
+ int err;
+
+ if (!HAS_XELINK(xe))
+ return;
+
+ if (!xe->xelink.present)
+ goto set_range;
+
+ err = init_irq_desc(xe);
+ if (err) {
+ xe->xelink.index = err;
+ goto set_range;
+ }
+
+ /*
+ * Try the socket id first. Systems with this feature, will
+ * get a deterministic value. If not, try with the cyclic.
+ */
+ err = xa_insert(&intel_fdevs, xe->xelink.socket_id, xe,
+ GFP_KERNEL);
+ if (!err)
+ index = xe->xelink.socket_id;
+
+ /* socket_id is not available */
+ if (err == -EBUSY) {
+ /*
+ * NOTE: index is only updated on success i.e. >= 0
+ * < 0 err, 0 ok, > 0 wrapped
+ */
+ err = xa_alloc_cyclic(&intel_fdevs, &index, xe,
+ XA_LIMIT(0, MAX_DEVICE_COUNT - 1),
+ &last_id, GFP_KERNEL);
+ }
+ if (err < 0) {
+ index = 0;
+ xe->xelink.index = err;
+ drm_err(&xe->drm,
+ "XeLink: Failed to allocate index: %d\n",
+ err);
+ irq_free_descs(xe->xelink.irq_base,
+ xe->info.tile_count);
+ goto set_range;
+ }
+ xe->xelink.index = index;
+ xe->xelink.dpa = (u64)index * MAX_DPA_SIZE * SZ_1G;
+ drm_info(&xe->drm, "XeLink: [dpa 0x%llx-0x%llx\n", xe->xelink.dpa,
+ ((u64)index + 1) * MAX_DPA_SIZE * SZ_1G - 1);
+
+ /*
+ * Set range has to be done for all devices that support device
+ * address space, regardless of presence or error.
+ */
+set_range:
+ /* Set GAM address range registers */
+ range = index * MAX_DPA_SIZE << PKG_ADDR_RANGE_BASE_SHIFT;
+ range |= MAX_DPA_SIZE << PKG_ADDR_RANGE_RANGE_SHIFT;
+ range |= PKG_ADDR_RANGE_ENABLE;
+
+ /* set SGunit address range register */
+ base = index * MAX_DPA_SIZE << PKG_ADDR_BASE_BASE_SHIFT;
+ base |= MAX_DPA_SIZE << PKG_ADDR_BASE_RANGE_SHIFT;
+ base |= PKG_ADDR_BASE_ENABLE;
+
+ /* Needs to be set for each gt */
+ for_each_gt(gt, xe, i) {
+ xe_mmio_write32(gt, PKG_ADDR_RANGE, range);
+ xe_mmio_write32(gt, PKG_ADDR_BASE, base);
+ }
+}
+
+static void xe_link_release_dev(struct device *dev)
+{
+ struct auxiliary_device *aux = to_auxiliary_dev(dev);
+ struct xelink_pdata *pd = container_of(aux, struct xelink_pdata, aux_dev);
+
+ kfree(pd->resources);
+ pd->resources = NULL;
+
+ kfree(pd);
+}
+
+/**
+ * xe_link_init_aux - Initialize resources and add auxiliary bus interface
+ * @xe: valid xe instance
+ *
+ */
+void xe_link_init_aux(struct xe_device *xe)
+{
+ struct device *dev = &to_pci_dev(xe->drm.dev)->dev;
+ struct resource *res = NULL;
+ struct xelink_pdata *pd;
+ int err = -ENOMEM;
+ u32 res_cnt;
+
+ if (!xe->xelink.present)
+ return;
+
+ if (xe->xelink.index < 0) {
+ err = xe->xelink.index;
+ goto fail;
+ }
+
+ pd = init_pd(xe);
+ if (!pd)
+ goto cleanup;
+
+ res = init_resource(xe, &res_cnt);
+ if (!res)
+ goto cleanup;
+
+ pd->resources = res;
+ pd->num_resources = res_cnt;
+
+ pd->aux_dev.name = "xelink";
+ pd->aux_dev.id = pd->index;
+ pd->aux_dev.dev.parent = dev;
+ pd->aux_dev.dev.release = xe_link_release_dev;
+
+ err = auxiliary_device_init(&pd->aux_dev);
+ if (err)
+ goto cleanup;
+
+ err = auxiliary_device_add(&pd->aux_dev);
+ if (err) {
+ auxiliary_device_uninit(&pd->aux_dev);
+ goto cleanup;
+ }
+
+ xe->xelink.pd = pd;
+
+ return;
+
+cleanup:
+ xa_erase(&intel_fdevs, xe->xelink.index);
+ irq_free_descs(xe->xelink.irq_base, xe->info.tile_count);
+ kfree(res);
+ kfree(pd);
+ xe->xelink.index = err;
+fail:
+ drm_err(&xe->drm, "XeLink: Failed to initialize err: %d\n", err);
+}
+
+void xe_link_remove(struct xe_device *xe)
+{
+ if (xe->xelink.index < 0)
+ return;
+
+ auxiliary_device_delete(&xe->xelink.pd->aux_dev);
+ auxiliary_device_uninit(&xe->xelink.pd->aux_dev);
+ xa_erase(&intel_fdevs, xe->xelink.index);
+ irq_free_descs(xe->xelink.irq_base, xe->info.tile_count);
+
+ xe->xelink.ops = &default_ops;
+}
diff --git a/drivers/gpu/drm/xe/xe_link.h b/drivers/gpu/drm/xe/xe_link.h
new file mode 100644
index 000000000000..099fc9e20550
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_link.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 - 2023 Intel Corporation
+ */
+
+#ifndef _XE_LINK_H_
+#define _XE_LINK_H_
+
+
+struct xe_device;
+
+void xe_link_init_early(struct xe_device *xe);
+void xe_link_init_mmio(struct xe_device *xe);
+void xe_link_init(struct xe_device *xe);
+void xe_link_init_aux(struct xe_device *xe);
+void xe_link_remove(struct xe_device *xe);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c
index f660cfb79f50..7529c6057e5e 100644
--- a/drivers/gpu/drm/xe/xe_mmio.c
+++ b/drivers/gpu/drm/xe/xe_mmio.c
@@ -153,8 +153,7 @@ static int xe_determine_lmem_bar_size(struct xe_device *xe)
if (!xe->mem.vram.io_size)
return -EIO;
- /* XXX: Need to change when xe link code is ready */
- xe->mem.vram.dpa_base = 0;
+ xe->mem.vram.dpa_base = xe->xelink.dpa;
/* set up a map to the total memory area. */
xe->mem.vram.mapping = ioremap_wc(xe->mem.vram.io_start, xe->mem.vram.io_size);
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index 1f997353a78f..be61eb84b7ac 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -148,6 +148,7 @@ static const struct xe_graphics_desc graphics_xehpc = {
.has_asid = 1,
.has_flat_ccs = 0,
.has_usm = 1,
+ .has_xelink = 1,
};
static const struct xe_graphics_desc graphics_xelpg = {
@@ -617,6 +618,7 @@ static int xe_info_init(struct xe_device *xe,
xe->info.va_bits = graphics_desc->va_bits;
xe->info.vm_max_level = graphics_desc->vm_max_level;
xe->info.has_asid = graphics_desc->has_asid;
+ xe->info.has_xelink = graphics_desc->has_xelink;
xe->info.has_flat_ccs = graphics_desc->has_flat_ccs;
xe->info.has_range_tlb_invalidation = graphics_desc->has_range_tlb_invalidation;
xe->info.has_usm = graphics_desc->has_usm;
diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h
index b1ad12fa22d6..d3b5f5766f07 100644
--- a/drivers/gpu/drm/xe/xe_pci_types.h
+++ b/drivers/gpu/drm/xe/xe_pci_types.h
@@ -25,6 +25,7 @@ struct xe_graphics_desc {
u8 max_remote_tiles:2;
u8 has_asid:1;
+ u8 has_xelink:1;
u8 has_flat_ccs:1;
u8 has_range_tlb_invalidation:1;
u8 has_usm:1;
diff --git a/include/drm/xelink_platform.h b/include/drm/xelink_platform.h
new file mode 100644
index 000000000000..81f33a38097a
--- /dev/null
+++ b/include/drm/xelink_platform.h
@@ -0,0 +1,140 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright(c) 2019 - 2023 Intel Corporation.
+ */
+
+#ifndef __XELINK_PLATFORM_H
+#define __XELINK_PLATFORM_H
+
+#define XELINK_VERSION 1
+
+#include <linux/auxiliary_bus.h>
+#include <linux/types.h>
+
+/**
+ * enum product_type - Product type identifying the parent
+ * @XELINK_UNDEFINED: no product type hints
+ * @XELINK_PONTEVECCHIO: parent is a PVC
+ * @XELINK_PRODUCTS: end of the list
+ *
+ */
+enum product_type {
+ XELINK_UNDEFINED,
+ XELINK_PONTEVECCHIO,
+ XELINK_PRODUCTS
+};
+
+/**
+ * enum xelink_dev_event - events generated to the parent device
+ * @XELINK_DEV_REMOVE: Xelink device was removed
+ * @XELINK_DEV_ERROR: An error occurred
+ * @XELINK_DEV_EVENTS: end of list
+ *
+ * Connectivity events, possible errors, etc.
+ */
+enum xelink_dev_event {
+ XELINK_DEV_REMOVE,
+ XELINK_DEV_EVENTS
+};
+
+/**
+ * enum xelink_parent_event - Events generated by the parent device
+ * @XELINK_PARENT_PCIE_ERR: Parent had a PCI error
+ * @XELINK_PARENT_MAPPING_GET: Notify XeLink of buffer mapping
+ * @XELINK_PARENT_MAPPING_PUT: Notify XeLink of buffer unmapping
+ *
+ * These are examples.
+ */
+enum xelink_parent_event {
+ XELINK_PARENT_PCIE_ERR,
+ XELINK_PARENT_MAPPING_GET,
+ XELINK_PARENT_MAPPING_PUT,
+};
+
+/**
+ * struct sd2sd_info - Subdevice to subdevice connectivity info
+ * @bandwidth: in Gbps units not factoring in width or quality degredation
+ * @latency: in 1/10 hops units
+ */
+struct sd2sd_info {
+ u16 bandwidth;
+ u16 latency;
+};
+
+/**
+ * struct query_info - connectivity query response information
+ * @src_cnt: Requester subdevice count
+ * @dst_cnt: Destination path count
+ * @sd2sd: array of src/dst bandwidth/latency information
+ *
+ * Query info will be a variably sized data structure allocated by the
+ * XeLink driver. The access will be indexed by
+ * (src_index * dst_cnt) + dst_index
+ *
+ * The caller will need to free the buffer when done.
+ */
+struct query_info {
+ u8 src_cnt;
+ u8 dst_cnt;
+ struct sd2sd_info sd2sd[];
+};
+
+/**
+ * struct xelink_ops - Communication path from parent to XeLink instance
+ * @connectivity_query: Query a device for xelink_id connectivity
+ * @parent_event: Any events needed by the XeLink device
+ *
+ * connectivity_query() returns:
+ * a populated query_info on success,
+ * an ERR_PTR() on failure
+ *
+ */
+struct xelink_ops {
+ struct query_info *(*connectivity_query)(void *handle, u32 xelink_id);
+ int (*parent_event)(void *handle, enum xelink_parent_event event);
+};
+
+struct dpa_space {
+ u32 pkg_offset;
+ u16 pkg_size;
+};
+
+/**
+ * struct xelink_pdata - Platform specific data that needs to be shared
+ * @version: PSD version information
+ * @parent: Handle to use when calling the parent device
+ * @product: a product hint for any necessary special case requirements
+ * @index: unique device index. This will be part of the device name
+ * @dpa: Device physical address offset and size
+ * @sd_cnt: parent subdevice count
+ * @socket_id: device socket information
+ * @slot: PCI/CXL slot number
+ * @aux_dev: Auxiliary bus device
+ * @resources: Array of resources (Assigned by Xe, the IRQ/MEM for the device)
+ * @num_resources: number of resources in resources array
+ * @register_dev: Register a XeLink instance and ops with the parent device
+ * @unregister_dev: Unregister a XeLink instance from the parent device
+ * @dev_event: Notify parent that an event has occurred
+ */
+struct xelink_pdata {
+ u16 version;
+ void *parent;
+ enum product_type product;
+ u16 index;
+ struct dpa_space dpa;
+ u8 sd_cnt;
+ u8 socket_id;
+ u8 slot;
+
+ struct auxiliary_device aux_dev;
+ struct resource *resources;
+ u32 num_resources;
+
+ int (*register_dev)(void *parent, void *handle, u32 xelink_id,
+ const struct xelink_ops *ops);
+ void (*unregister_dev)(void *parent, const void *handle);
+ int (*dev_event)(void *parent, void *handle,
+ enum xelink_dev_event event, void *event_data);
+};
+
+#endif /* __XELINK_PLATFORM_H */
--
2.38.1
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH v4 2/3] drm/xe: Teach Xe how to use objects with XeLink connectivity
2023-12-20 23:17 [PATCH v4 0/3] Add support for XeLink device David Kershner
2023-12-20 23:17 ` [PATCH v4 1/3] drm/xe: Introduce " David Kershner
@ 2023-12-20 23:17 ` David Kershner
2023-12-20 23:17 ` [PATCH v4 3/3] drm/xe/uapi: Augment query ioctl to allow for fabric David Kershner
2 siblings, 0 replies; 4+ messages in thread
From: David Kershner @ 2023-12-20 23:17 UTC (permalink / raw)
To: david.kershner, intel-xe, michael.j.ruhl, john.fleck,
lucas.demarchi, rodrigo.vivi, matthew.d.roper, jani.nikula
The XeLink device offers fabric backed memory. Page table entries
need to be aware of this fact so that the access occur over the
fabric rather than to the system or local device memory.
Objects are shared via dma-buf, and then fabric connectivity is
checked during the _get_pages() pass.
Objects do NOT need to be dma mapped because system dma does not
occur.
The Device Physical Address (DPA) is the memory range assigned
to the fabric device at driver init. This address is what will
need to be programmed into the page table entry. In addition
the PTE_LM bit needs to be set.
Address information is provided in a scatter/gather table.
Teach the dma-buf interface and the page table entries all about
fabric backed memory.
Signed-off-by: David Kershner <david.kershner@intel.com>
---
drivers/gpu/drm/xe/xe_bo.c | 46 +++++-
drivers/gpu/drm/xe/xe_bo.h | 2 +
drivers/gpu/drm/xe/xe_dma_buf.c | 209 +++++++++++++++++++--------
drivers/gpu/drm/xe/xe_dma_buf.h | 3 +
drivers/gpu/drm/xe/xe_ggtt.c | 3 +
drivers/gpu/drm/xe/xe_link.c | 12 ++
drivers/gpu/drm/xe/xe_link.h | 2 +
drivers/gpu/drm/xe/xe_pt.c | 2 +
drivers/gpu/drm/xe/xe_trace.h | 29 ++++
drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 17 ++-
drivers/gpu/drm/xe/xe_vm.c | 3 +-
11 files changed, 258 insertions(+), 70 deletions(-)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 8e4a3b1f6b93..8b30ba47b447 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -8,6 +8,7 @@
#include <linux/dma-buf.h>
#include <drm/drm_drv.h>
+#include <drm/drm_gem.h>
#include <drm/drm_gem_ttm_helper.h>
#include <drm/drm_managed.h>
#include <drm/ttm/ttm_device.h>
@@ -20,6 +21,7 @@
#include "xe_drm_client.h"
#include "xe_ggtt.h"
#include "xe_gt.h"
+#include "xe_link.h"
#include "xe_map.h"
#include "xe_migrate.h"
#include "xe_preempt_fence.h"
@@ -521,6 +523,32 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
return ret;
}
+/**
+ * map_xelink_connectivity - check for XeLink and create a mappable sgt
+ * if available
+ * @bo: object to check XeLink connectivity
+ *
+ * Returns sgt or -errno on error, -EIO indicates no XeLink connectivity.
+ */
+static struct sg_table *map_xelink_connectivity(struct xe_bo *bo)
+{
+ struct dma_buf_attachment *attach = bo->ttm.base.import_attach;
+ struct xe_bo *import;
+
+ if (!(bo->flags & XE_BO_XELINK_AVAIL))
+ return ERR_PTR(-EIO);
+
+ import = gem_to_xe_bo(attach->dmabuf->priv);
+
+ /* Make sure the object didn't migrate */
+ if (!xe_bo_is_vram(import)) {
+ bo->flags &= ~XE_BO_XELINK_AVAIL;
+ return ERR_PTR(-EIO);
+ }
+
+ return xe_dma_buf_map(attach, DMA_NONE);
+}
+
/*
* The dma-buf map_attachment() / unmap_attachment() is hooked up here.
* Note that unmapping the attachment is deferred to the next
@@ -538,6 +566,7 @@ static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,
ttm);
struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
struct sg_table *sg;
+ struct xe_bo *bo, *export;
xe_assert(xe, attach);
xe_assert(xe, ttm_bo->ttm);
@@ -545,12 +574,25 @@ static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,
if (new_res->mem_type == XE_PL_SYSTEM)
goto out;
+ bo = ttm_to_xe_bo(ttm_bo);
if (ttm_bo->sg) {
- dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
+ if (bo->flags & XE_BO_XELINK_AVAIL) {
+ export = gem_to_xe_bo(attach->dmabuf->priv);
+ xe_link_mapping_put(to_xe_device(export->ttm.base.dev));
+ xe_link_mapping_put(to_xe_device(ttm_bo->base.dev));
+ sg_free_table(ttm_bo->sg);
+ kfree(ttm_bo->sg);
+ } else {
+ dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
+ }
ttm_bo->sg = NULL;
}
- sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+ sg = map_xelink_connectivity(bo);
+
+ if (IS_ERR(sg) && PTR_ERR(sg) == -EIO)
+ sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+
if (IS_ERR(sg))
return PTR_ERR(sg);
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 97b32528c600..48b8f27099e3 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -45,6 +45,8 @@
#define XE_BO_PAGETABLE BIT(12)
#define XE_BO_NEEDS_CPU_ACCESS BIT(13)
#define XE_BO_NEEDS_UC BIT(14)
+#define XE_BO_XELINK_AVAIL BIT(15)
+
/* this one is trigger internally only */
#define XE_BO_INTERNAL_TEST BIT(30)
#define XE_BO_INTERNAL_64K BIT(31)
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index 64ed303728fd..bfff479ac1f2 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -11,36 +11,35 @@
#include <drm/drm_device.h>
#include <drm/drm_prime.h>
+#include <drm/xelink_platform.h>
#include <drm/ttm/ttm_tt.h>
#include "tests/xe_test.h"
#include "xe_bo.h"
#include "xe_device.h"
+#include "xe_link.h"
#include "xe_ttm_vram_mgr.h"
+#include "xe_trace.h"
#include "xe_vm.h"
MODULE_IMPORT_NS(DMA_BUF);
-static int xe_dma_buf_attach(struct dma_buf *dmabuf,
- struct dma_buf_attachment *attach)
-{
- struct drm_gem_object *obj = attach->dmabuf->priv;
-
- if (attach->peer2peer &&
- pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0)
- attach->peer2peer = false;
-
- if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
- return -EOPNOTSUPP;
-
- xe_device_mem_access_get(to_xe_device(obj->dev));
- return 0;
-}
static void xe_dma_buf_detach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attach)
{
struct drm_gem_object *obj = attach->dmabuf->priv;
+ struct xe_bo *bo = gem_to_xe_bo(obj);
+
+ if (attach->importer_priv) {
+ struct drm_gem_object *im_obj = attach->importer_priv;
+ struct xe_bo *im_bo = gem_to_xe_bo(im_obj);
+
+ if (im_bo->flags & XE_BO_XELINK_AVAIL) {
+ xe_link_mapping_put(xe_bo_device(bo));
+ xe_link_mapping_put(xe_bo_device(im_bo));
+ }
+ }
xe_device_mem_access_put(to_xe_device(obj->dev));
}
@@ -85,20 +84,29 @@ static void xe_dma_buf_unpin(struct dma_buf_attachment *attach)
xe_bo_unpin_external(bo);
}
-static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
- enum dma_data_direction dir)
+struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
+ enum dma_data_direction dir)
{
struct dma_buf *dma_buf = attach->dmabuf;
struct drm_gem_object *obj = dma_buf->priv;
struct xe_bo *bo = gem_to_xe_bo(obj);
struct sg_table *sgt;
int r = 0;
+ bool xelink_avail = false;
+
+ if (attach->importer_priv) {
+ struct drm_gem_object *im_obj = attach->importer_priv;
+ struct xe_bo *im_bo = gem_to_xe_bo(im_obj);
- if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
+ xelink_avail = im_bo->flags & XE_BO_XELINK_AVAIL;
+ }
+
+ if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT) &&
+ !xelink_avail)
return ERR_PTR(-EOPNOTSUPP);
if (!xe_bo_is_pinned(bo)) {
- if (!attach->peer2peer)
+ if (!attach->peer2peer && !xelink_avail)
r = xe_bo_migrate(bo, XE_PL_TT);
else
r = xe_bo_validate(bo, NULL, false);
@@ -175,41 +183,12 @@ static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
return 0;
}
-const struct dma_buf_ops xe_dmabuf_ops = {
- .attach = xe_dma_buf_attach,
- .detach = xe_dma_buf_detach,
- .pin = xe_dma_buf_pin,
- .unpin = xe_dma_buf_unpin,
- .map_dma_buf = xe_dma_buf_map,
- .unmap_dma_buf = xe_dma_buf_unmap,
- .release = drm_gem_dmabuf_release,
- .begin_cpu_access = xe_dma_buf_begin_cpu_access,
- .mmap = drm_gem_dmabuf_mmap,
- .vmap = drm_gem_dmabuf_vmap,
- .vunmap = drm_gem_dmabuf_vunmap,
-};
-
-struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
-{
- struct xe_bo *bo = gem_to_xe_bo(obj);
- struct dma_buf *buf;
-
- if (bo->vm)
- return ERR_PTR(-EPERM);
-
- buf = drm_gem_prime_export(obj, flags);
- if (!IS_ERR(buf))
- buf->ops = &xe_dmabuf_ops;
-
- return buf;
-}
static struct drm_gem_object *
-xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
+xe_dma_buf_init_obj(struct xe_device *xe, struct xe_bo *storage,
struct dma_buf *dma_buf)
{
struct dma_resv *resv = dma_buf->resv;
- struct xe_device *xe = to_xe_device(dev);
struct xe_bo *bo;
int ret;
@@ -256,6 +235,121 @@ struct dma_buf_test_params {
container_of(_priv, struct dma_buf_test_params, base)
#endif
+static int xe_dma_buf_attach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach);
+
+const struct dma_buf_ops xe_dmabuf_ops = {
+ .attach = xe_dma_buf_attach,
+ .detach = xe_dma_buf_detach,
+ .pin = xe_dma_buf_pin,
+ .unpin = xe_dma_buf_unpin,
+ .map_dma_buf = xe_dma_buf_map,
+ .unmap_dma_buf = xe_dma_buf_unmap,
+ .release = drm_gem_dmabuf_release,
+ .begin_cpu_access = xe_dma_buf_begin_cpu_access,
+ .mmap = drm_gem_dmabuf_mmap,
+ .vmap = drm_gem_dmabuf_vmap,
+ .vunmap = drm_gem_dmabuf_vunmap,
+};
+
+/*
+ * update_xelink - check for XeLink connectivity if available
+ * @obj: object to check XeLink connectivity
+ *
+ * If the imported object is a i915 dma-buf, and LMEM based, query to see if
+ * there is a XeLink, and if the XeLink is connected set the XeLink bit.
+ *
+ * 0 no connectivity, use P2P if available
+ * 1 XeLink is available
+ * -1 XeLink only is requested, and there is no XeLink
+ *
+ */
+static int update_xelink(struct dma_buf *dma_buf, struct xe_bo *bo)
+{
+ struct xe_bo *import;
+ struct xe_device *src;
+ struct xe_device *dst;
+ struct query_info *qi;
+ int connected;
+ int i;
+ int n;
+
+ /* Verify that both sides are xe devices */
+ if (dma_buf->ops != &xe_dmabuf_ops || !bo)
+ return 0;
+
+ import = gem_to_xe_bo(dma_buf->priv);
+ if (!xe_bo_is_vram(import))
+ return 0;
+
+ src = xe_bo_device(bo);
+ dst = xe_bo_device(import);
+
+ qi = src->xelink.ops->connectivity_query(src->xelink.handle,
+ dst->xelink.xelink_id);
+ if (IS_ERR(qi))
+ return 0;
+
+ /*
+ * Examine the query information. A zero bandwidth link indicates we
+ * are NOT connected.
+ */
+ connected = 1;
+ for (i = 0, n = qi->src_cnt * qi->dst_cnt; i < n && connected; i++)
+ if (!qi->sd2sd[i].bandwidth)
+ connected = 0;
+
+ /* we are responsible for freeing qi */
+ kfree(qi);
+
+ if (connected) {
+ if (xe_link_mapping_get(src))
+ return 0;
+ if (xe_link_mapping_get(dst)) {
+ xe_link_mapping_put(src);
+ return 0;
+ }
+ bo->flags |= XE_BO_XELINK_AVAIL;
+ }
+
+ return connected;
+}
+
+static int xe_dma_buf_attach(struct dma_buf *dmabuf,
+ struct dma_buf_attachment *attach)
+{
+ struct drm_gem_object *obj = attach->dmabuf->priv;
+ struct xe_bo *bo = gem_to_xe_bo(attach->importer_priv);
+ int xelink;
+
+ xelink = update_xelink(dmabuf, bo);
+
+ if (attach->peer2peer &&
+ pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0)
+ attach->peer2peer = false;
+
+ trace_xe_dma_buf_attach(gem_to_xe_bo(obj), bo, xelink, attach->peer2peer);
+ if (!xelink && !attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
+ return -EOPNOTSUPP;
+
+ xe_device_mem_access_get(to_xe_device(obj->dev));
+ return 0;
+}
+
+struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
+{
+ struct xe_bo *bo = gem_to_xe_bo(obj);
+ struct dma_buf *buf;
+
+ if (bo->vm)
+ return ERR_PTR(-EPERM);
+
+ buf = drm_gem_prime_export(obj, flags);
+ if (!IS_ERR(buf))
+ buf->ops = &xe_dmabuf_ops;
+
+ return buf;
+}
+
struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
struct dma_buf *dma_buf)
{
@@ -264,6 +358,7 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
(xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));)
const struct dma_buf_attach_ops *attach_ops;
struct dma_buf_attachment *attach;
+ struct xe_device *xe = to_xe_device(dev);
struct drm_gem_object *obj;
struct xe_bo *bo;
@@ -295,26 +390,18 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
attach_ops = test->attach_ops;
#endif
- attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base);
- if (IS_ERR(attach)) {
- obj = ERR_CAST(attach);
- goto out_err;
- }
-
/* Errors here will take care of freeing the bo. */
- obj = xe_dma_buf_init_obj(dev, bo, dma_buf);
+ obj = xe_dma_buf_init_obj(xe, bo, dma_buf);
if (IS_ERR(obj))
return obj;
+ attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base);
+ if (IS_ERR(attach))
+ return ERR_CAST(attach);
get_dma_buf(dma_buf);
obj->import_attach = attach;
return obj;
-
-out_err:
- xe_bo_free(bo);
-
- return obj;
}
#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.h b/drivers/gpu/drm/xe/xe_dma_buf.h
index 861dd28a862c..36771a50451e 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.h
+++ b/drivers/gpu/drm/xe/xe_dma_buf.h
@@ -7,9 +7,12 @@
#define _XE_DMA_BUF_H_
#include <drm/drm_gem.h>
+#include <linux/dma-direction.h>
struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags);
struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
struct dma_buf *dma_buf);
+struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
+ enum dma_data_direction dir);
#endif
diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index c639dbf3bdd2..2041a94ba80a 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -36,6 +36,9 @@ static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
pte |= XE_GGTT_PTE_DM;
+ if (xe_bo_is_vram(bo) || bo->flags & XE_BO_XELINK_AVAIL)
+ pte |= XE_PPGTT_PTE_DM;
+
return pte;
}
diff --git a/drivers/gpu/drm/xe/xe_link.c b/drivers/gpu/drm/xe/xe_link.c
index f1484fbb630d..ea6c10498b77 100644
--- a/drivers/gpu/drm/xe/xe_link.c
+++ b/drivers/gpu/drm/xe/xe_link.c
@@ -464,3 +464,15 @@ void xe_link_remove(struct xe_device *xe)
xe->xelink.ops = &default_ops;
}
+
+int xe_link_mapping_get(struct xe_device *xe)
+{
+ return xe->xelink.ops->parent_event(xe->xelink.handle,
+ XELINK_PARENT_MAPPING_GET);
+}
+
+int xe_link_mapping_put(struct xe_device *xe)
+{
+ return xe->xelink.ops->parent_event(xe->xelink.handle,
+ XELINK_PARENT_MAPPING_PUT);
+}
diff --git a/drivers/gpu/drm/xe/xe_link.h b/drivers/gpu/drm/xe/xe_link.h
index 099fc9e20550..a443f674d2d1 100644
--- a/drivers/gpu/drm/xe/xe_link.h
+++ b/drivers/gpu/drm/xe/xe_link.h
@@ -13,6 +13,8 @@ void xe_link_init_early(struct xe_device *xe);
void xe_link_init_mmio(struct xe_device *xe);
void xe_link_init(struct xe_device *xe);
void xe_link_init_aux(struct xe_device *xe);
+int xe_link_mapping_get(struct xe_device *xe);
+int xe_link_mapping_put(struct xe_device *xe);
void xe_link_remove(struct xe_device *xe);
#endif
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index de1030a47588..0731cc009689 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -609,6 +609,8 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
if (is_devmem) {
xe_walk.default_pte |= XE_PPGTT_PTE_DM;
xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource);
+ } else if (!xe_vma_has_no_bo(vma) && bo->flags & XE_BO_XELINK_AVAIL) {
+ xe_walk.default_pte |= XE_PPGTT_PTE_DM;
}
if (!xe_vma_has_no_bo(vma) && xe_bo_is_stolen(bo))
diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h
index 95163c303f3e..2ae9091c7e76 100644
--- a/drivers/gpu/drm/xe/xe_trace.h
+++ b/drivers/gpu/drm/xe/xe_trace.h
@@ -105,6 +105,35 @@ DEFINE_EVENT(xe_bo, xe_bo_move,
TP_ARGS(bo)
);
+DECLARE_EVENT_CLASS(xe_dma_buf,
+ TP_PROTO(struct xe_bo *bo, struct xe_bo *imp_bo, int xelink, int p2p),
+ TP_ARGS(bo, imp_bo, xelink, p2p),
+
+ TP_STRUCT__entry(
+ __field(struct xe_bo *, bo)
+ __field(struct xe_bo *, imp_bo)
+ __field(bool, xelink)
+ __field(bool, p2p)
+ ),
+
+ TP_fast_assign(
+ __entry->bo = bo;
+ __entry->imp_bo = imp_bo;
+ __entry->xelink = xelink;
+ __entry->p2p = p2p;
+ ),
+
+ TP_printk("bo=: 0x%p-mem_type=%d, imp_bo: 0x%p-mem_type=%d, xelink=%d, p2p=%d",
+ __entry->bo, __entry->bo->ttm.resource->mem_type,
+ __entry->imp_bo, __entry->imp_bo->ttm.resource->mem_type,
+ __entry->xelink, __entry->p2p)
+);
+
+DEFINE_EVENT(xe_dma_buf, xe_dma_buf_attach,
+ TP_PROTO(struct xe_bo *bo, struct xe_bo *imp_bo, int xelink, int p2p),
+ TP_ARGS(bo, imp_bo, xelink, p2p)
+);
+
DECLARE_EVENT_CLASS(xe_exec_queue,
TP_PROTO(struct xe_exec_queue *q),
TP_ARGS(q),
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
index 115ec745e502..71cb4ee9dce2 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
@@ -412,15 +412,20 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
*/
xe_res_first(res, offset, length, &cursor);
for_each_sgtable_sg((*sgt), sg, i) {
- phys_addr_t phys = cursor.start + tile->mem.vram.io_start;
size_t size = cursor.size;
dma_addr_t addr;
- addr = dma_map_resource(dev, phys, size, dir,
- DMA_ATTR_SKIP_CPU_SYNC);
- r = dma_mapping_error(dev, addr);
- if (r)
- goto error_unmap;
+ if (dir == DMA_NONE) {
+ addr = cursor.start + tile->mem.vram.dpa_base;
+ } else {
+ phys_addr_t phys = cursor.start + tile->mem.vram.io_start;
+
+ addr = dma_map_resource(dev, phys, size, dir,
+ DMA_ATTR_SKIP_CPU_SYNC);
+ r = dma_mapping_error(dev, addr);
+ if (r)
+ goto error_unmap;
+ }
sg_set_page(sg, NULL, size, 0);
sg_dma_address(sg) = addr;
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 1ca917b8315c..80cdead34488 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1135,7 +1135,8 @@ static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
pte |= pte_encode_pat_index(xe, pat_index, pt_level);
pte |= pte_encode_ps(pt_level);
- if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
+ if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo) ||
+ bo->flags & XE_BO_XELINK_AVAIL)
pte |= XE_PPGTT_PTE_DM;
return pte;
--
2.38.1
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH v4 3/3] drm/xe/uapi: Augment query ioctl to allow for fabric
2023-12-20 23:17 [PATCH v4 0/3] Add support for XeLink device David Kershner
2023-12-20 23:17 ` [PATCH v4 1/3] drm/xe: Introduce " David Kershner
2023-12-20 23:17 ` [PATCH v4 2/3] drm/xe: Teach Xe how to use objects with XeLink connectivity David Kershner
@ 2023-12-20 23:17 ` David Kershner
2 siblings, 0 replies; 4+ messages in thread
From: David Kershner @ 2023-12-20 23:17 UTC (permalink / raw)
To: david.kershner, intel-xe, michael.j.ruhl, john.fleck,
lucas.demarchi, rodrigo.vivi, matthew.d.roper, jani.nikula
UMDs need to understand if two devices have connectivity, and what
that connectivity is.
Add to the query_ioctl the ability to determine if a fabric id and
current device have connectivity.
Signed-off-by: David Kershner <david.kershner@intel.com>
---
drivers/gpu/drm/xe/xe_query.c | 54 +++++++++++++++++++++++++++++++++++
include/uapi/drm/xe_drm.h | 26 +++++++++++++++++
2 files changed, 80 insertions(+)
diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c
index 9b35673b286c..b7366acfcfd8 100644
--- a/drivers/gpu/drm/xe/xe_query.c
+++ b/drivers/gpu/drm/xe/xe_query.c
@@ -6,10 +6,12 @@
#include "xe_query.h"
#include <linux/nospec.h>
+#include <linux/overflow.h>
#include <linux/sched/clock.h>
#include <drm/ttm/ttm_placement.h>
#include <drm/xe_drm.h>
+#include <drm/xelink_platform.h>
#include "regs/xe_engine_regs.h"
#include "xe_bo.h"
@@ -104,6 +106,57 @@ __read_timestamps(struct xe_gt *gt,
*engine_ts = (u64)upper << 32 | lower;
}
+static int
+query_fabric_connectivity(struct xe_device *xe,
+ struct drm_xe_device_query *query)
+{
+ struct drm_xe_query_fabric_info __user *query_ptr;
+ struct drm_xe_query_fabric_info info;
+ struct query_info *qi;
+ u32 latency = 0;
+ int cnt;
+ int i;
+
+ query_ptr = u64_to_user_ptr(query->data);
+ if (copy_from_user(&info, query_ptr, sizeof(info)))
+ return -EFAULT;
+
+ info.bandwidth = 0;
+ info.latency = 0;
+
+ if (info.fabric_id == xe->xelink.xelink_id)
+ goto done;
+
+ qi = xe->xelink.ops->connectivity_query(xe->xelink.handle, info.fabric_id);
+ if (IS_ERR(qi))
+ goto done;
+ /*
+ * Examine the query information for connectivity.
+ * Minimum bandwidth value is the bandwidth, 0 == no connectivity
+ * Latency is averaged.
+ */
+ cnt = qi->src_cnt * qi->dst_cnt;
+ if (!cnt) {
+ kfree(qi);
+ return -ENXIO;
+ }
+
+ info.bandwidth = 0xffff;
+ for (i = 0; i < cnt; i++) {
+ info.bandwidth = min(qi->sd2sd[i].bandwidth, info.bandwidth);
+ XE_WARN_ON(check_add_overflow(latency, qi->sd2sd[i].latency,
+ &latency));
+ }
+
+ info.latency = latency / cnt;
+
+ kfree(qi);
+done:
+ if (copy_to_user(query_ptr, &info, sizeof(info)))
+ return -EFAULT;
+ return 0;
+}
+
static int
query_engine_cycles(struct xe_device *xe,
struct drm_xe_device_query *query)
@@ -529,6 +582,7 @@ static int (* const xe_query_funcs[])(struct xe_device *xe,
query_hwconfig,
query_gt_topology,
query_engine_cycles,
+ query_fabric_connectivity,
};
int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
index 9fa3ae324731..b2d4af6729e1 100644
--- a/include/uapi/drm/xe_drm.h
+++ b/include/uapi/drm/xe_drm.h
@@ -574,6 +574,31 @@ struct drm_xe_query_engine_cycles {
__u64 cpu_delta;
};
+/**
+ * struct drm_xe_query_fabric_info
+ *
+ * With the given fabric id, query fabric info wrt the device.
+ * Higher bandwidth is better. 0 means no fabric.
+ * Latency is averaged latency (from all paths)
+ *
+ * fabric_id can be obtained from
+ * /sys/class/drm/cardx/device/xelink.y/xelink_fabric_id
+ * Bandwidth is in Gigabits per second (max value of 8 * 4 * 90)
+ * 8 possible ports
+ * 4 lanes max per port
+ * 90 gigabits per lane
+ * Latency is in tenths of path length. 10 == 1 fabric link between src and dst
+ * POR is max 1 link (zero hops).
+ */
+struct drm_xe_query_fabric_info {
+ /** @fabric_id: Fabric id associated with info */
+ __u32 fabric_id;
+ /** @bandwidth: minimum bandwidth of all connected ports, if 0 no fabric */
+ __u16 bandwidth;
+ /** @latency: latency averaged across all connected ports. */
+ __u16 latency;
+};
+
/**
* struct drm_xe_device_query - Input of &DRM_IOCTL_XE_DEVICE_QUERY - main
* structure to query device information
@@ -643,6 +668,7 @@ struct drm_xe_device_query {
#define DRM_XE_DEVICE_QUERY_HWCONFIG 4
#define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5
#define DRM_XE_DEVICE_QUERY_ENGINE_CYCLES 6
+#define DRM_XE_DEVICE_QUERY_FABRIC_INFO 7
/** @query: The type of data to query */
__u32 query;
--
2.38.1
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2023-12-20 23:17 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2023-12-20 23:17 [PATCH v4 0/3] Add support for XeLink device David Kershner
2023-12-20 23:17 ` [PATCH v4 1/3] drm/xe: Introduce " David Kershner
2023-12-20 23:17 ` [PATCH v4 2/3] drm/xe: Teach Xe how to use objects with XeLink connectivity David Kershner
2023-12-20 23:17 ` [PATCH v4 3/3] drm/xe/uapi: Augment query ioctl to allow for fabric David Kershner
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox