[Patch v9 18/18] net/mana: support Rx interrupts

[longli@linuxonhyperv.com]

From: Long Li <longli@microsoft.com>

mana can receive Rx interrupts from kernel through RDMA verbs interface.
Implement Rx interrupts in the driver.

Signed-off-by: Long Li <longli@microsoft.com>
---
Change log:
v5:
New patch added to the series
v8:
Fix coding style on function definitions.

 doc/guides/nics/features/mana.ini |   1 +
 drivers/net/mana/gdma.c           |  10 +--
 drivers/net/mana/mana.c           | 128 ++++++++++++++++++++++++++----
 drivers/net/mana/mana.h           |   9 ++-
 drivers/net/mana/rx.c             |  94 +++++++++++++++++++---
 drivers/net/mana/tx.c             |   3 +-
 6 files changed, 211 insertions(+), 34 deletions(-)

diff --git a/doc/guides/nics/features/mana.ini b/doc/guides/nics/features/mana.ini
index 81ebc9c365..5fb62ea85d 100644
--- a/doc/guides/nics/features/mana.ini
+++ b/doc/guides/nics/features/mana.ini
@@ -14,6 +14,7 @@ Multiprocess aware   = Y
 Queue start/stop     = Y
 Removal event        = Y
 RSS hash             = Y
+Rx interrupt         = Y
 Speed capabilities   = P
 Usage doc            = Y
 x86-64               = Y
diff --git a/drivers/net/mana/gdma.c b/drivers/net/mana/gdma.c
index 3f937d6c93..c67c5af2f9 100644
--- a/drivers/net/mana/gdma.c
+++ b/drivers/net/mana/gdma.c
@@ -213,7 +213,7 @@ union gdma_doorbell_entry {
  */
 int
 mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
-		   uint32_t queue_id, uint32_t tail)
+		   uint32_t queue_id, uint32_t tail, uint8_t arm)
 {
 	uint8_t *addr = db_page;
 	union gdma_doorbell_entry e = {};
@@ -228,14 +228,14 @@ mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
 	case GDMA_QUEUE_RECEIVE:
 		e.rq.id = queue_id;
 		e.rq.tail_ptr = tail;
-		e.rq.wqe_cnt = 1;
+		e.rq.wqe_cnt = arm;
 		addr += DOORBELL_OFFSET_RQ;
 		break;
 
 	case GDMA_QUEUE_COMPLETION:
 		e.cq.id = queue_id;
 		e.cq.tail_ptr = tail;
-		e.cq.arm = 1;
+		e.cq.arm = arm;
 		addr += DOORBELL_OFFSET_CQ;
 		break;
 
@@ -247,8 +247,8 @@ mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
 	/* Ensure all writes are done before ringing doorbell */
 	rte_wmb();
 
-	DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u",
-		db_page, addr, queue_id, queue_type, tail);
+	DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u arm %u",
+		db_page, addr, queue_id, queue_type, tail, arm);
 
 	rte_write64(e.as_uint64, addr);
 	return 0;
diff --git a/drivers/net/mana/mana.c b/drivers/net/mana/mana.c
index d247b930db..d277a35dae 100644
--- a/drivers/net/mana/mana.c
+++ b/drivers/net/mana/mana.c
@@ -103,7 +103,72 @@ mana_dev_configure(struct rte_eth_dev *dev)
 	return 0;
 }
 
-static int mana_intr_uninstall(struct mana_priv *priv);
+static void
+rx_intr_vec_disable(struct mana_priv *priv)
+{
+	struct rte_intr_handle *intr_handle = priv->intr_handle;
+
+	rte_intr_free_epoll_fd(intr_handle);
+	rte_intr_vec_list_free(intr_handle);
+	rte_intr_nb_efd_set(intr_handle, 0);
+}
+
+static int
+rx_intr_vec_enable(struct mana_priv *priv)
+{
+	unsigned int i;
+	unsigned int rxqs_n = priv->dev_data->nb_rx_queues;
+	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
+	struct rte_intr_handle *intr_handle = priv->intr_handle;
+	int ret;
+
+	rx_intr_vec_disable(priv);
+
+	if (rte_intr_vec_list_alloc(intr_handle, NULL, n)) {
+		DRV_LOG(ERR, "Failed to allocate memory for interrupt vector");
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < n; i++) {
+		struct mana_rxq *rxq = priv->dev_data->rx_queues[i];
+
+		ret = rte_intr_vec_list_index_set(intr_handle, i,
+						  RTE_INTR_VEC_RXTX_OFFSET + i);
+		if (ret) {
+			DRV_LOG(ERR, "Failed to set intr vec %u", i);
+			return ret;
+		}
+
+		ret = rte_intr_efds_index_set(intr_handle, i, rxq->channel->fd);
+		if (ret) {
+			DRV_LOG(ERR, "Failed to set FD at intr %u", i);
+			return ret;
+		}
+	}
+
+	return rte_intr_nb_efd_set(intr_handle, n);
+}
+
+static void
+rxq_intr_disable(struct mana_priv *priv)
+{
+	int err = rte_errno;
+
+	rx_intr_vec_disable(priv);
+	rte_errno = err;
+}
+
+static int
+rxq_intr_enable(struct mana_priv *priv)
+{
+	const struct rte_eth_intr_conf *const intr_conf =
+		&priv->dev_data->dev_conf.intr_conf;
+
+	if (!intr_conf->rxq)
+		return 0;
+
+	return rx_intr_vec_enable(priv);
+}
 
 static int
 mana_dev_start(struct rte_eth_dev *dev)
@@ -141,8 +206,17 @@ mana_dev_start(struct rte_eth_dev *dev)
 	/* Enable datapath for secondary processes */
 	mana_mp_req_on_rxtx(dev, MANA_MP_REQ_START_RXTX);
 
+	ret = rxq_intr_enable(priv);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to enable RX interrupts");
+		goto failed_intr;
+	}
+
 	return 0;
 
+failed_intr:
+	mana_stop_rx_queues(dev);
+
 failed_rx:
 	mana_stop_tx_queues(dev);
 
@@ -153,9 +227,12 @@ mana_dev_start(struct rte_eth_dev *dev)
 }
 
 static int
-mana_dev_stop(struct rte_eth_dev *dev __rte_unused)
+mana_dev_stop(struct rte_eth_dev *dev)
 {
 	int ret;
+	struct mana_priv *priv = dev->data->dev_private;
+
+	rxq_intr_disable(priv);
 
 	dev->tx_pkt_burst = mana_tx_burst_removed;
 	dev->rx_pkt_burst = mana_rx_burst_removed;
@@ -180,6 +257,8 @@ mana_dev_stop(struct rte_eth_dev *dev __rte_unused)
 	return 0;
 }
 
+static int mana_intr_uninstall(struct mana_priv *priv);
+
 static int
 mana_dev_close(struct rte_eth_dev *dev)
 {
@@ -614,6 +693,8 @@ static const struct eth_dev_ops mana_dev_ops = {
 	.tx_queue_release	= mana_dev_tx_queue_release,
 	.rx_queue_setup		= mana_dev_rx_queue_setup,
 	.rx_queue_release	= mana_dev_rx_queue_release,
+	.rx_queue_intr_enable	= mana_rx_intr_enable,
+	.rx_queue_intr_disable	= mana_rx_intr_disable,
 	.link_update		= mana_dev_link_update,
 	.stats_get		= mana_dev_stats_get,
 	.stats_reset		= mana_dev_stats_reset,
@@ -849,10 +930,22 @@ mana_intr_uninstall(struct mana_priv *priv)
 	return 0;
 }
 
+int
+mana_fd_set_non_blocking(int fd)
+{
+	int ret = fcntl(fd, F_GETFL);
+
+	if (ret != -1 && !fcntl(fd, F_SETFL, ret | O_NONBLOCK))
+		return 0;
+
+	rte_errno = errno;
+	return -rte_errno;
+}
+
 static int
-mana_intr_install(struct mana_priv *priv)
+mana_intr_install(struct rte_eth_dev *eth_dev, struct mana_priv *priv)
 {
-	int ret, flags;
+	int ret;
 	struct ibv_context *ctx = priv->ib_ctx;
 
 	priv->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
@@ -862,31 +955,35 @@ mana_intr_install(struct mana_priv *priv)
 		return -ENOMEM;
 	}
 
-	rte_intr_fd_set(priv->intr_handle, -1);
+	ret = rte_intr_fd_set(priv->intr_handle, -1);
+	if (ret)
+		goto free_intr;
 
-	flags = fcntl(ctx->async_fd, F_GETFL);
-	ret = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+	ret = mana_fd_set_non_blocking(ctx->async_fd);
 	if (ret) {
 		DRV_LOG(ERR, "Failed to change async_fd to NONBLOCK");
 		goto free_intr;
 	}
 
-	rte_intr_fd_set(priv->intr_handle, ctx->async_fd);
-	rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT);
+	ret = rte_intr_fd_set(priv->intr_handle, ctx->async_fd);
+	if (ret)
+		goto free_intr;
+
+	ret = rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT);
+	if (ret)
+		goto free_intr;
 
 	ret = rte_intr_callback_register(priv->intr_handle,
 					 mana_intr_handler, priv);
 	if (ret) {
 		DRV_LOG(ERR, "Failed to register intr callback");
 		rte_intr_fd_set(priv->intr_handle, -1);
-		goto restore_fd;
+		goto free_intr;
 	}
 
+	eth_dev->intr_handle = priv->intr_handle;
 	return 0;
 
-restore_fd:
-	fcntl(ctx->async_fd, F_SETFL, flags);
-
 free_intr:
 	rte_intr_instance_free(priv->intr_handle);
 	priv->intr_handle = NULL;
@@ -1224,8 +1321,10 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev,
 				name, priv->max_rx_queues, priv->max_rx_desc,
 				priv->max_send_sge);
 
+			rte_eth_copy_pci_info(eth_dev, pci_dev);
+
 			/* Create async interrupt handler */
-			ret = mana_intr_install(priv);
+			ret = mana_intr_install(eth_dev, priv);
 			if (ret) {
 				DRV_LOG(ERR, "Failed to install intr handler");
 				goto failed;
@@ -1246,7 +1345,6 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev,
 			eth_dev->tx_pkt_burst = mana_tx_burst_removed;
 			eth_dev->dev_ops = &mana_dev_ops;
 
-			rte_eth_copy_pci_info(eth_dev, pci_dev);
 			rte_eth_dev_probing_finish(eth_dev);
 		}
 
diff --git a/drivers/net/mana/mana.h b/drivers/net/mana/mana.h
index 9c576a82fa..14d813466e 100644
--- a/drivers/net/mana/mana.h
+++ b/drivers/net/mana/mana.h
@@ -425,6 +425,7 @@ struct mana_rxq {
 	uint32_t num_desc;
 	struct rte_mempool *mp;
 	struct ibv_cq *cq;
+	struct ibv_comp_channel *channel;
 	struct ibv_wq *wq;
 
 	/* For storing pending requests */
@@ -458,8 +459,8 @@ extern int mana_logtype_init;
 #define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
 
 int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
-		       uint32_t queue_id, uint32_t tail);
-int mana_rq_ring_doorbell(struct mana_rxq *rxq);
+		       uint32_t queue_id, uint32_t tail, uint8_t arm);
+int mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm);
 
 int gdma_post_work_request(struct mana_gdma_queue *queue,
 			   struct gdma_work_request *work_req,
@@ -539,4 +540,8 @@ void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type);
 void *mana_alloc_verbs_buf(size_t size, void *data);
 void mana_free_verbs_buf(void *ptr, void *data __rte_unused);
 
+int mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int mana_fd_set_non_blocking(int fd);
+
 #endif
diff --git a/drivers/net/mana/rx.c b/drivers/net/mana/rx.c
index b80a5d1c7a..57dfae7bcd 100644
--- a/drivers/net/mana/rx.c
+++ b/drivers/net/mana/rx.c
@@ -22,7 +22,7 @@ static uint8_t mana_rss_hash_key_default[TOEPLITZ_HASH_KEY_SIZE_IN_BYTES] = {
 };
 
 int
-mana_rq_ring_doorbell(struct mana_rxq *rxq)
+mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm)
 {
 	struct mana_priv *priv = rxq->priv;
 	int ret;
@@ -37,9 +37,9 @@ mana_rq_ring_doorbell(struct mana_rxq *rxq)
 	}
 
 	ret = mana_ring_doorbell(db_page, GDMA_QUEUE_RECEIVE,
-				 rxq->gdma_rq.id,
-				 rxq->gdma_rq.head *
-					GDMA_WQE_ALIGNMENT_UNIT_SIZE);
+			 rxq->gdma_rq.id,
+			 rxq->gdma_rq.head * GDMA_WQE_ALIGNMENT_UNIT_SIZE,
+			 arm);
 
 	if (ret)
 		DRV_LOG(ERR, "failed to ring RX doorbell ret %d", ret);
@@ -121,7 +121,7 @@ mana_alloc_and_post_rx_wqes(struct mana_rxq *rxq)
 		}
 	}
 
-	mana_rq_ring_doorbell(rxq);
+	mana_rq_ring_doorbell(rxq, rxq->num_desc);
 
 	return ret;
 }
@@ -163,6 +163,14 @@ mana_stop_rx_queues(struct rte_eth_dev *dev)
 				DRV_LOG(ERR,
 					"rx_queue destroy_cq failed %d", ret);
 			rxq->cq = NULL;
+
+			if (rxq->channel) {
+				ret = ibv_destroy_comp_channel(rxq->channel);
+				if (ret)
+					DRV_LOG(ERR, "failed destroy comp %d",
+						ret);
+				rxq->channel = NULL;
+			}
 		}
 
 		/* Drain and free posted WQEs */
@@ -204,8 +212,24 @@ mana_start_rx_queues(struct rte_eth_dev *dev)
 				.data = (void *)(uintptr_t)rxq->socket,
 			}));
 
+		if (dev->data->dev_conf.intr_conf.rxq) {
+			rxq->channel = ibv_create_comp_channel(priv->ib_ctx);
+			if (!rxq->channel) {
+				ret = -errno;
+				DRV_LOG(ERR, "Queue %d comp channel failed", i);
+				goto fail;
+			}
+
+			ret = mana_fd_set_non_blocking(rxq->channel->fd);
+			if (ret) {
+				DRV_LOG(ERR, "Failed to set comp non-blocking");
+				goto fail;
+			}
+		}
+
 		rxq->cq = ibv_create_cq(priv->ib_ctx, rxq->num_desc,
-					NULL, NULL, 0);
+					NULL, rxq->channel,
+					rxq->channel ? i : 0);
 		if (!rxq->cq) {
 			ret = -errno;
 			DRV_LOG(ERR, "failed to create rx cq queue %d", i);
@@ -356,7 +380,8 @@ mana_start_rx_queues(struct rte_eth_dev *dev)
 uint16_t
 mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
-	uint16_t pkt_received = 0, cqe_processed = 0;
+	uint16_t pkt_received = 0;
+	uint8_t wqe_posted = 0;
 	struct mana_rxq *rxq = dpdk_rxq;
 	struct mana_priv *priv = rxq->priv;
 	struct gdma_comp comp;
@@ -442,18 +467,65 @@ mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		if (rxq->desc_ring_tail >= rxq->num_desc)
 			rxq->desc_ring_tail = 0;
 
-		cqe_processed++;
-
 		/* Post another request */
 		ret = mana_alloc_and_post_rx_wqe(rxq);
 		if (ret) {
 			DRV_LOG(ERR, "failed to post rx wqe ret=%d", ret);
 			break;
 		}
+
+		wqe_posted++;
 	}
 
-	if (cqe_processed)
-		mana_rq_ring_doorbell(rxq);
+	if (wqe_posted)
+		mana_rq_ring_doorbell(rxq, wqe_posted);
 
 	return pkt_received;
 }
+
+static int
+mana_arm_cq(struct mana_rxq *rxq, uint8_t arm)
+{
+	struct mana_priv *priv = rxq->priv;
+	uint32_t head = rxq->gdma_cq.head %
+		(rxq->gdma_cq.count << COMPLETION_QUEUE_ENTRY_OWNER_BITS_SIZE);
+
+	DRV_LOG(ERR, "Ringing completion queue ID %u head %u arm %d",
+		rxq->gdma_cq.id, head, arm);
+
+	return mana_ring_doorbell(priv->db_page, GDMA_QUEUE_COMPLETION,
+				  rxq->gdma_cq.id, head, arm);
+}
+
+int
+mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id];
+
+	return mana_arm_cq(rxq, 1);
+}
+
+int
+mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id];
+	struct ibv_cq *ev_cq;
+	void *ev_ctx;
+	int ret;
+
+	ret = ibv_get_cq_event(rxq->channel, &ev_cq, &ev_ctx);
+	if (ret)
+		ret = errno;
+	else if (ev_cq != rxq->cq)
+		ret = EINVAL;
+
+	if (ret) {
+		if (ret != EAGAIN)
+			DRV_LOG(ERR, "Can't disable RX intr queue %d",
+				rx_queue_id);
+	} else {
+		ibv_ack_cq_events(rxq->cq, 1);
+	}
+
+	return -ret;
+}
diff --git a/drivers/net/mana/tx.c b/drivers/net/mana/tx.c
index 0884681c30..a92d895e54 100644
--- a/drivers/net/mana/tx.c
+++ b/drivers/net/mana/tx.c
@@ -406,7 +406,8 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		ret = mana_ring_doorbell(db_page, GDMA_QUEUE_SEND,
 					 txq->gdma_sq.id,
 					 txq->gdma_sq.head *
-						GDMA_WQE_ALIGNMENT_UNIT_SIZE);
+						GDMA_WQE_ALIGNMENT_UNIT_SIZE,
+					 0);
 	if (ret)
 		DRV_LOG(ERR, "mana_ring_doorbell failed ret %d", ret);
 
-- 
2.17.1