[PATCH 0/4] vdpa/octeon_ep: fixes and improvements for multi VF

[PATCH 0/4] vdpa/octeon_ep: fixes and improvements for multi VF

[Srujana Challa]

This series fixes several issues in the Marvell OCTEON DPU vDPA driver
and adds support for firmware-initiated device events.

Patch 1 fixes the PF->VF mailbox address calculation when multiple
rings per VF are configured. The driver previously assumed 1 ring per
VF; it now reads the actual rings-per-VF from OCTEP_EPF_RINFO.

Patch 2 switches mailbox signature access from 64-bit to 32-bit, since
the upper 4 bytes are reserved by firmware for metadata.

Patch 3 adds handling for vDPA device add and remove events from
Octeon firmware, using irq 0 for multiplexed event delivery.

Patch 4 fixes the IRQ-to-ring mapping in the interrupt handler. The
original logic assumed contiguous IRQs starting at irqs[0]; the driver
now looks up the IRQ index in oct_hw->irqs to support non-contiguous
IRQ numbers and avoid incorrect ring indexing.

Srujana Challa (2):
  vdpa/octeon_ep: Fix PF->VF mailbox data address calculation
  vdpa/octeon_ep: fix IRQ-to-ring mapping in interrupt handler

Vamsi Attunuru (2):
  vdpa/octeon_ep: Use 4 bytes for mailbox signature
  vdpa/octeon_ep: Add vDPA device event handling for firmware
    notifications

 drivers/vdpa/octeon_ep/octep_vdpa.h      |  22 +++-
 drivers/vdpa/octeon_ep/octep_vdpa_main.c | 131 +++++++++++++++++++----
 2 files changed, 134 insertions(+), 19 deletions(-)

-- 
2.25.1

[PATCH 1/4] vdpa/octeon_ep: Fix PF->VF mailbox data address calculation

[Srujana Challa]

The mailbox address was computed assuming 1 ring per VF. Read the
actual rings-per-VF from OCTEP_EPF_RINFO and use it when calculating
OCTEP_PF_MBOX_DATA offsets, fixing VF initialization when rings
per VF > 1.

Fixes: 8b6c724cdab8 ("virtio: vdpa: vDPA driver for Marvell OCTEON DPU devices")
Signed-off-by: Srujana Challa <schalla@marvell.com>
---
 drivers/vdpa/octeon_ep/octep_vdpa_main.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
index 31a02e7fd7f2..9946480ee704 100644
--- a/drivers/vdpa/octeon_ep/octep_vdpa_main.c
+++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /* Copyright (C) 2024 Marvell. */
 
+#include <linux/bitfield.h>
 #include <linux/interrupt.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/module.h>
@@ -722,6 +723,8 @@ static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs)
 	bool done = false;
 	int index = 0;
 	int ret, i;
+	u8 rpvf;
+	u64 val;
 
 	ret = pci_enable_sriov(pdev, num_vfs);
 	if (ret)
@@ -741,9 +744,11 @@ static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs)
 		}
 	}
 
+	val = readq(addr + OCTEP_EPF_RINFO(0));
+	rpvf = FIELD_GET(GENMASK_ULL(35, 32), val);
 	if (done) {
 		for (i = 0; i < pf->enabled_vfs; i++)
-			writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i));
+			writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i * rpvf));
 	}
 
 	return num_vfs;
-- 
2.25.1

[PATCH 2/4] vdpa/octeon_ep: Use 4 bytes for mailbox signature

[Srujana Challa]

From: Vamsi Attunuru <vattunuru@marvell.com>

The upper 4 bytes are reserved by the firmware for
storing meta data. Use only lower 4 bytes to update
the signature details.

Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
---
 drivers/vdpa/octeon_ep/octep_vdpa_main.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
index 9946480ee704..deaa8dc7813e 100644
--- a/drivers/vdpa/octeon_ep/octep_vdpa_main.c
+++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
@@ -573,10 +573,10 @@ static const struct vdpa_mgmtdev_ops octep_vdpa_mgmt_dev_ops = {
 
 static bool get_device_ready_status(u8 __iomem *addr)
 {
-	u64 signature = readq(addr + OCTEP_VF_MBOX_DATA(0));
+	u32 signature = readl(addr + OCTEP_VF_MBOX_DATA(0));
 
 	if (signature == OCTEP_DEV_READY_SIGNATURE) {
-		writeq(0, addr + OCTEP_VF_MBOX_DATA(0));
+		writel(0, addr + OCTEP_VF_MBOX_DATA(0));
 		return true;
 	}
 
@@ -748,7 +748,7 @@ static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs)
 	rpvf = FIELD_GET(GENMASK_ULL(35, 32), val);
 	if (done) {
 		for (i = 0; i < pf->enabled_vfs; i++)
-			writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i * rpvf));
+			writel(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i * rpvf));
 	}
 
 	return num_vfs;
-- 
2.25.1

[PATCH 3/4] vdpa/octeon_ep: Add vDPA device event handling for firmware notifications

[Srujana Challa]

From: Vamsi Attunuru <vattunuru@marvell.com>

Handle vDPA device add and remove events from Octeon firmware. Use
irq 0 for event delivery as device interrupts are multiplexed.

Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
---
 drivers/vdpa/octeon_ep/octep_vdpa.h      |  22 ++++-
 drivers/vdpa/octeon_ep/octep_vdpa_main.c | 107 ++++++++++++++++++++---
 2 files changed, 115 insertions(+), 14 deletions(-)

diff --git a/drivers/vdpa/octeon_ep/octep_vdpa.h b/drivers/vdpa/octeon_ep/octep_vdpa.h
index 53b020b019f7..a67bf50e4075 100644
--- a/drivers/vdpa/octeon_ep/octep_vdpa.h
+++ b/drivers/vdpa/octeon_ep/octep_vdpa.h
@@ -30,8 +30,10 @@
 #define OCTEP_EPF_RINFO(x) (0x000209f0 | ((x) << 25))
 #define OCTEP_VF_MBOX_DATA(x) (0x00010210 | ((x) << 17))
 #define OCTEP_PF_MBOX_DATA(x) (0x00022000 | ((x) << 4))
+#define OCTEP_VF_EVENT_STATE(x) (0x00010030 | ((x) << 17))
+#define OCTEP_VF_EVENT_REG(x) (0x00010060 | ((x) << 17))
 #define OCTEP_VF_IN_CTRL(x)        (0x00010000 | ((x) << 17))
-#define OCTEP_VF_IN_CTRL_RPVF(val) (((val) >> 48) & 0xF)
+#define OCTEP_VF_IN_CTRL_RPVF(val) (FIELD_GET(GENMASK_ULL(51, 48), val))
 
 #define OCTEP_FW_READY_SIGNATURE0  0xFEEDFEED
 #define OCTEP_FW_READY_SIGNATURE1  0x3355ffaa
@@ -43,9 +45,26 @@ enum octep_vdpa_dev_status {
 	OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT,
 	OCTEP_VDPA_DEV_STATUS_INIT,
 	OCTEP_VDPA_DEV_STATUS_READY,
+	OCTEP_VDPA_DEV_STATUS_ADDED,
+	OCTEP_VDPA_DEV_STATUS_REMOVED,
 	OCTEP_VDPA_DEV_STATUS_UNINIT
 };
 
+enum octep_vdpa_dev_event_state {
+	OCTEP_VDPA_DEV_NO_EVENT,
+	OCTEP_VDPA_DEV_NEW_EVENT,
+	OCTEP_VDPA_DEV_EVENT_ACTIVE,
+	OCTEP_VDPA_DEV_EVENT_DONE,
+};
+
+enum octep_vdpa_dev_event {
+	OCTEP_VDPA_DEV_EVENT_NONE,
+	OCTEP_VDPA_DEV_EVENT_ACK,
+	OCTEP_VDPA_DEV_EVENT_NACK,
+	OCTEP_VDPA_DEV_ADD_EVENT,
+	OCTEP_VDPA_DEV_DEL_EVENT,
+};
+
 struct octep_vring_info {
 	struct vdpa_callback cb;
 	void __iomem *notify_addr;
@@ -86,6 +105,7 @@ struct octep_hw {
 	u64 features;
 	u16 nr_vring;
 	u32 config_size;
+	int requested_irqs;
 	int nb_irqs;
 	int *irqs;
 	u8 dev_id;
diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
index deaa8dc7813e..5b9d76632376 100644
--- a/drivers/vdpa/octeon_ep/octep_vdpa_main.c
+++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
@@ -9,6 +9,7 @@
 #include "octep_vdpa.h"
 
 #define OCTEP_VDPA_DRIVER_NAME "octep_vdpa"
+#define OCTEP_VDPA_NAME_BUFSIZE 16
 
 struct octep_pf {
 	u8 __iomem *base[PCI_STD_NUM_BARS];
@@ -20,6 +21,11 @@ struct octep_pf {
 	u16 vf_devid;
 };
 
+struct octep_vdpa_event_wk {
+	struct work_struct work;
+	void *ctxptr;
+};
+
 struct octep_vdpa {
 	struct vdpa_device vdpa;
 	struct octep_hw *oct_hw;
@@ -34,6 +40,8 @@ struct octep_vdpa_mgmt_dev {
 	struct work_struct setup_task;
 	/* Device status */
 	atomic_t status;
+	struct octep_vdpa *oct_vdpa;
+	struct octep_vdpa_event_wk event_wk;
 };
 
 static struct octep_hw *vdpa_to_octep_hw(struct vdpa_device *vdpa_dev)
@@ -45,6 +53,27 @@ static struct octep_hw *vdpa_to_octep_hw(struct vdpa_device *vdpa_dev)
 	return oct_vdpa->oct_hw;
 }
 
+static inline void octep_vdpa_dev_event_schedule(struct octep_hw *oct_hw)
+{
+	u8 __iomem *addr = oct_hw->base[OCTEP_HW_MBOX_BAR];
+	struct octep_vdpa_mgmt_dev *mgmt_dev;
+
+	mgmt_dev = container_of(oct_hw, struct octep_vdpa_mgmt_dev, oct_hw);
+	writeb(OCTEP_VDPA_DEV_EVENT_ACTIVE, addr + OCTEP_VF_EVENT_STATE(0));
+	schedule_work(&mgmt_dev->event_wk.work);
+}
+
+static irqreturn_t octep_vdpa_dev_event_handler(int irq, void *data)
+{
+	struct octep_hw *oct_hw = data;
+
+	if (readb(oct_hw->base[OCTEP_HW_MBOX_BAR] + OCTEP_VF_EVENT_STATE(0)) ==
+	    OCTEP_VDPA_DEV_NEW_EVENT)
+		octep_vdpa_dev_event_schedule(oct_hw);
+
+	return IRQ_HANDLED;
+}
+
 static irqreturn_t octep_vdpa_intr_handler(int irq, void *data)
 {
 	struct octep_hw *oct_hw = data;
@@ -73,11 +102,14 @@ static irqreturn_t octep_vdpa_intr_handler(int irq, void *data)
 	}
 
 	/* Check for config interrupt. Config uses the first interrupt */
-	if (unlikely(irq == oct_hw->irqs[0] && ioread8(oct_hw->isr))) {
-		iowrite8(0, oct_hw->isr);
+	if (unlikely(irq == oct_hw->irqs[0])) {
+		if (ioread8(oct_hw->isr)) {
+			iowrite8(0, oct_hw->isr);
 
-		if (oct_hw->config_cb.callback)
-			oct_hw->config_cb.callback(oct_hw->config_cb.private);
+			if (oct_hw->config_cb.callback)
+				oct_hw->config_cb.callback(oct_hw->config_cb.private);
+		}
+		octep_vdpa_dev_event_handler(irq, data);
 	}
 
 	return IRQ_HANDLED;
@@ -101,33 +133,41 @@ static void octep_free_irqs(struct octep_hw *oct_hw)
 	pci_free_irq_vectors(pdev);
 	devm_kfree(&pdev->dev, oct_hw->irqs);
 	oct_hw->irqs = NULL;
+	oct_hw->requested_irqs = 0;
 }
 
-static int octep_request_irqs(struct octep_hw *oct_hw)
+static int octep_request_irqs(struct octep_hw *oct_hw, irqreturn_t (*irq_handler)(int, void *),
+			      int nb_irqs)
 {
 	struct pci_dev *pdev = oct_hw->pdev;
 	int ret, irq, idx;
 
-	oct_hw->irqs = devm_kcalloc(&pdev->dev, oct_hw->nb_irqs, sizeof(int), GFP_KERNEL);
+	if ((oct_hw->requested_irqs != nb_irqs) || (nb_irqs == 1))
+		octep_free_irqs(oct_hw);
+	else
+		return 0;
+
+	oct_hw->irqs = devm_kcalloc(&pdev->dev, nb_irqs, sizeof(int), GFP_KERNEL);
 	if (!oct_hw->irqs)
 		return -ENOMEM;
 
-	ret = pci_alloc_irq_vectors(pdev, 1, oct_hw->nb_irqs, PCI_IRQ_MSIX);
+	ret = pci_alloc_irq_vectors(pdev, 1, nb_irqs, PCI_IRQ_MSIX);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to alloc msix vector");
 		return ret;
 	}
 
-	for (idx = 0; idx < oct_hw->nb_irqs; idx++) {
+	for (idx = 0; idx < nb_irqs; idx++) {
 		irq = pci_irq_vector(pdev, idx);
-		ret = devm_request_irq(&pdev->dev, irq, octep_vdpa_intr_handler, 0,
-				       dev_name(&pdev->dev), oct_hw);
+		ret = devm_request_irq(&pdev->dev, irq, irq_handler, 0, dev_name(&pdev->dev),
+				       oct_hw);
 		if (ret) {
 			dev_err(&pdev->dev, "Failed to register interrupt handler\n");
 			goto free_irqs;
 		}
 		oct_hw->irqs[idx] = irq;
 	}
+	oct_hw->requested_irqs = nb_irqs;
 
 	return 0;
 
@@ -189,7 +229,7 @@ static void octep_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
 
 	if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 	    !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
-		if (octep_request_irqs(oct_hw))
+		if (octep_request_irqs(oct_hw, octep_vdpa_intr_handler, oct_hw->nb_irqs))
 			status = status_old | VIRTIO_CONFIG_S_FAILED;
 	}
 	octep_hw_set_status(oct_hw, status);
@@ -212,8 +252,10 @@ static int octep_vdpa_reset(struct vdpa_device *vdpa_dev)
 	}
 	octep_hw_reset(oct_hw);
 
-	if (status & VIRTIO_CONFIG_S_DRIVER_OK)
+	if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
 		octep_free_irqs(oct_hw);
+		octep_request_irqs(oct_hw, octep_vdpa_dev_event_handler, 1);
+	}
 
 	return 0;
 }
@@ -478,7 +520,8 @@ static void octep_vdpa_remove_vf(struct pci_dev *pdev)
 	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_UNINIT);
 
 	cancel_work_sync(&mgmt_dev->setup_task);
-	if (status == OCTEP_VDPA_DEV_STATUS_READY)
+	if ((status == OCTEP_VDPA_DEV_STATUS_READY) || (status == OCTEP_VDPA_DEV_STATUS_ADDED) ||
+	    (status == OCTEP_VDPA_DEV_STATUS_REMOVED))
 		vdpa_mgmtdev_unregister(&mgmt_dev->mdev);
 
 	if (oct_hw->base[OCTEP_HW_CAPS_BAR])
@@ -488,6 +531,7 @@ static void octep_vdpa_remove_vf(struct pci_dev *pdev)
 		octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_MBOX_BAR);
 
 	octep_vdpa_vf_bar_shrink(pdev);
+	octep_free_irqs(oct_hw);
 }
 
 static void octep_vdpa_remove(struct pci_dev *pdev)
@@ -521,6 +565,7 @@ static int octep_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
 	oct_vdpa->vdpa.mdev = mdev;
 	oct_vdpa->oct_hw = oct_hw;
 	vdpa_dev = &oct_vdpa->vdpa;
+	mgmt_dev->oct_vdpa = oct_vdpa;
 
 	device_features = oct_hw->features;
 	if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
@@ -554,6 +599,7 @@ static int octep_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
 		dev_err(&pdev->dev, "Failed to register to vDPA bus");
 		goto vdpa_dev_put;
 	}
+	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_ADDED);
 	return 0;
 
 vdpa_dev_put:
@@ -563,7 +609,9 @@ static int octep_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
 
 static void octep_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *vdpa_dev)
 {
+	struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(mdev, struct octep_vdpa_mgmt_dev, mdev);
 	_vdpa_unregister_device(vdpa_dev);
+	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_REMOVED);
 }
 
 static const struct vdpa_mgmtdev_ops octep_vdpa_mgmt_dev_ops = {
@@ -588,6 +636,36 @@ static struct virtio_device_id id_table[] = {
 	{ 0 },
 };
 
+static void octep_event_work(struct work_struct *work)
+{
+	struct octep_vdpa_event_wk *wk = container_of(work, struct octep_vdpa_event_wk, work);
+	struct octep_vdpa_mgmt_dev *mgmt_dev = (struct octep_vdpa_mgmt_dev *)wk->ctxptr;
+	u8 __iomem *addr = mgmt_dev->oct_hw.base[OCTEP_HW_MBOX_BAR];
+	u8 event = readb(addr + OCTEP_VF_EVENT_REG(0));
+	struct vdpa_dev_set_config config = {0};
+	char name[OCTEP_VDPA_NAME_BUFSIZE];
+	int ret = 0;
+
+	switch (event) {
+	case OCTEP_VDPA_DEV_ADD_EVENT:
+		if (atomic_read(&mgmt_dev->status) != OCTEP_VDPA_DEV_STATUS_ADDED) {
+			snprintf(name, sizeof(name), "%s-%x", "vdpa", mgmt_dev->pdev->devfn);
+			ret = octep_vdpa_dev_add(&mgmt_dev->mdev, name, &config);
+		}
+		break;
+	case OCTEP_VDPA_DEV_DEL_EVENT:
+		if (atomic_read(&mgmt_dev->status) == OCTEP_VDPA_DEV_STATUS_ADDED)
+			octep_vdpa_dev_del(&mgmt_dev->mdev, &mgmt_dev->oct_vdpa->vdpa);
+		break;
+	default:
+		break;
+	}
+
+	event = ret ? OCTEP_VDPA_DEV_EVENT_NACK : OCTEP_VDPA_DEV_EVENT_ACK;
+	writeb(event, addr + OCTEP_VF_EVENT_REG(0));
+	writeb(OCTEP_VDPA_DEV_EVENT_DONE, addr + OCTEP_VF_EVENT_STATE(0));
+}
+
 static void octep_vdpa_setup_task(struct work_struct *work)
 {
 	struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(work, struct octep_vdpa_mgmt_dev,
@@ -653,6 +731,9 @@ static void octep_vdpa_setup_task(struct work_struct *work)
 	}
 
 	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_READY);
+	INIT_WORK(&mgmt_dev->event_wk.work, octep_event_work);
+	mgmt_dev->event_wk.ctxptr = mgmt_dev;
+	octep_request_irqs(&mgmt_dev->oct_hw, octep_vdpa_dev_event_handler, 1);
 
 	return;
 
-- 
2.25.1

[PATCH 4/4] vdpa/octeon_ep: fix IRQ-to-ring mapping in interrupt handler

[Srujana Challa]

Look up the IRQ index in oct_hw->irqs instead of assuming
irq - irqs[0]. This supports non-contiguous IRQ numbers and
avoids incorrect ring indexing when irqs[0] is not the base.

Fixes: 26f8ce06af64 ("vdpa/octeon_ep: enable support for multiple interrupts per device")
Signed-off-by: Srujana Challa <schalla@marvell.com>
---
 drivers/vdpa/octeon_ep/octep_vdpa_main.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
index 5b9d76632376..5b35993750f5 100644
--- a/drivers/vdpa/octeon_ep/octep_vdpa_main.c
+++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c
@@ -77,7 +77,7 @@ static irqreturn_t octep_vdpa_dev_event_handler(int irq, void *data)
 static irqreturn_t octep_vdpa_intr_handler(int irq, void *data)
 {
 	struct octep_hw *oct_hw = data;
-	int i;
+	int i, start_ring_idx = -1;
 
 	/* Each device has multiple interrupts (nb_irqs) shared among rings
 	 * (nr_vring). Device interrupts are mapped to the rings in a
@@ -90,7 +90,16 @@ static irqreturn_t octep_vdpa_intr_handler(int irq, void *data)
 	 * 7 -> 7, 15, 23, 31, 39, 47, 55, 63;
 	 */
 
-	for (i = irq - oct_hw->irqs[0]; i < oct_hw->nr_vring; i += oct_hw->nb_irqs) {
+	for (i = 0; i < oct_hw->nb_irqs; i++) {
+		if (oct_hw->irqs[i] == irq) {
+			start_ring_idx = i;
+			break;
+		}
+	}
+	if (start_ring_idx == -1)
+		return IRQ_NONE;
+
+	for (i = start_ring_idx; i < oct_hw->nr_vring; i += oct_hw->nb_irqs) {
 		if (ioread8(oct_hw->vqs[i].cb_notify_addr)) {
 			/* Acknowledge the per ring notification to the device */
 			iowrite8(0, oct_hw->vqs[i].cb_notify_addr);
-- 
2.25.1