[PATCH 1/2] net/mlx5: distinguish DevX availability from DevX use

[PATCH 1/2] net/mlx5: distinguish DevX availability from DevX use

[Maayan Kashani]

mlx5_devx_obj_ops_en() required DV flow and Dest TIR in addition to
DevX. Dest TIR is never set on Windows, so common code treated DevX
ports as Verbs. LRO stayed disabled although the device supports it.

Split the two questions. mlx5_devx_available() reports whether DevX
exists. mlx5_devx_obj_ops_en() reports whether the port selected DevX
object operations, using a flag in struct mlx5_obj_ops. Linux still
requires DV flow and Dest TIR when choosing those operations.

Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c    | 56 ++++++++++++++---------------
 drivers/net/mlx5/linux/mlx5_verbs.c |  1 +
 drivers/net/mlx5/mlx5.c             |  2 +-
 drivers/net/mlx5/mlx5.h             | 33 ++++++++++-------
 drivers/net/mlx5/mlx5_devx.c        |  3 +-
 drivers/net/mlx5/mlx5_ethdev.c      |  2 +-
 drivers/net/mlx5/mlx5_trigger.c     |  2 +-
 drivers/net/mlx5/mlx5_txq.c         | 10 +++---
 8 files changed, 59 insertions(+), 50 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 7c2c35c9454..4c357372929 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1446,31 +1446,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		err = ENOMEM;
 		goto error;
 	}
-	/*
-	 * When user configures remote PD and CTX and device creates RxQ by
-	 * DevX, external RxQ is both supported and requested.
-	 */
-	if (mlx5_imported_pd_and_ctx(sh->cdev) && mlx5_devx_obj_ops_en(sh)) {
-		priv->ext_rxqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
-					     sizeof(struct mlx5_external_q) *
-					     MLX5_MAX_EXT_RX_QUEUES, 0,
-					     SOCKET_ID_ANY);
-		if (priv->ext_rxqs == NULL) {
-			DRV_LOG(ERR, "Fail to allocate external RxQ array.");
-			err = ENOMEM;
-			goto error;
-		}
-		priv->ext_txqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
-					     sizeof(struct mlx5_external_q) *
-					     MLX5_MAX_EXT_TX_QUEUES, 0,
-					     SOCKET_ID_ANY);
-		if (priv->ext_txqs == NULL) {
-			DRV_LOG(ERR, "Fail to allocate external TxQ array.");
-			err = ENOMEM;
-			goto error;
-		}
-		DRV_LOG(DEBUG, "External queue is supported.");
-	}
 	priv->sh = sh;
 	priv->dev_port = spawn->phys_port;
 	priv->pci_dev = spawn->pci_dev;
@@ -1822,7 +1797,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
-	if (mlx5_devx_obj_ops_en(sh)) {
+	/* Linux DevX object operations also require DV flow and Dest TIR. */
+	if (mlx5_devx_available(sh) && sh->config.dv_flow_en && sh->dev_cap.dest_tir) {
 		priv->obj_ops = mlx5_devx_obj_ops;
 		mlx5_queue_counter_id_prepare(eth_dev);
 		priv->obj_ops.lb_dummy_queue_create =
@@ -1837,8 +1813,32 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	} else {
 		priv->obj_ops = mlx5_ibv_obj_ops;
 	}
-	if (sh->config.tx_pp &&
-	    priv->obj_ops.txq_obj_new != mlx5_txq_devx_obj_new) {
+	/*
+	 * When user configures remote PD and CTX and device creates RxQ by
+	 * DevX, external RxQ is both supported and requested.
+	 */
+	if (mlx5_imported_pd_and_ctx(sh->cdev) && mlx5_devx_obj_ops_en(priv)) {
+		priv->ext_rxqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
+					     sizeof(struct mlx5_external_q) *
+					     MLX5_MAX_EXT_RX_QUEUES, 0,
+					     SOCKET_ID_ANY);
+		if (priv->ext_rxqs == NULL) {
+			DRV_LOG(ERR, "Fail to allocate external RxQ array.");
+			err = ENOMEM;
+			goto error;
+		}
+		priv->ext_txqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
+					     sizeof(struct mlx5_external_q) *
+					     MLX5_MAX_EXT_TX_QUEUES, 0,
+					     SOCKET_ID_ANY);
+		if (priv->ext_txqs == NULL) {
+			DRV_LOG(ERR, "Fail to allocate external TxQ array.");
+			err = ENOMEM;
+			goto error;
+		}
+		DRV_LOG(DEBUG, "External queue is supported.");
+	}
+	if (sh->config.tx_pp && !mlx5_devx_obj_ops_en(priv)) {
 		/*
 		 * HAVE_MLX5DV_DEVX_UAR_OFFSET is required to support
 		 * packet pacing and already checked above.
diff --git a/drivers/net/mlx5/linux/mlx5_verbs.c b/drivers/net/mlx5/linux/mlx5_verbs.c
index 103a03bd1ab..ce0a26c62bf 100644
--- a/drivers/net/mlx5/linux/mlx5_verbs.c
+++ b/drivers/net/mlx5/linux/mlx5_verbs.c
@@ -1242,4 +1242,5 @@ struct mlx5_obj_ops mlx5_ibv_obj_ops = {
 	.txq_obj_release = mlx5_txq_ibv_obj_release,
 	.lb_dummy_queue_create = NULL,
 	.lb_dummy_queue_release = NULL,
+	.devx = false,
 };
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 19491b9dc95..e2dbd2977b7 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1588,7 +1588,7 @@ mlx5_shared_dev_ctx_args_config(struct mlx5_dev_ctx_shared *sh,
 		config->tx_typical_pkt_sz = 0;
 	}
 	/* Check for LRO support. */
-	if (mlx5_devx_obj_ops_en(sh) && sh->cdev->config.hca_attr.lro_cap) {
+	if (mlx5_devx_available(sh) && sh->cdev->config.hca_attr.lro_cap) {
 		/* TBD check tunnel lro caps. */
 		config->lro_allowed = 1;
 		DRV_LOG(DEBUG, "LRO is allowed.");
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e3df458915c..7923125c5cb 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1862,6 +1862,7 @@ struct mlx5_obj_ops {
 	void (*txq_obj_release)(struct mlx5_txq_obj *txq_obj);
 	int (*lb_dummy_queue_create)(struct rte_eth_dev *dev);
 	void (*lb_dummy_queue_release)(struct rte_eth_dev *dev);
+	bool devx; /* True when these are DevX object operations. */
 };
 
 enum mlx5_ctrl_flow_type {
@@ -2241,27 +2242,35 @@ mlx5_flow_action_job_init(struct mlx5_priv *priv, uint32_t queue,
 #endif
 
 /**
- * Indicates whether HW objects operations can be created by DevX.
+ * Indicates whether DevX is available on the device.
  *
- * This function is used for both:
- *  Before creation - deciding whether to create HW objects operations by DevX.
- *  After creation - indicator if HW objects operations were created by DevX.
+ * Availability does not mean DevX object operations were selected.
  *
  * @param sh
  *   Pointer to shared device context.
  *
  * @return
- *   True if HW objects were created by DevX, False otherwise.
+ *   True if DevX is available, false otherwise.
  */
 static inline bool
-mlx5_devx_obj_ops_en(struct mlx5_dev_ctx_shared *sh)
+mlx5_devx_available(struct mlx5_dev_ctx_shared *sh)
 {
-	/*
-	 * When advanced DR API is available and DV flow is supported and
-	 * DevX is supported, HW objects operations are created by DevX.
-	 */
-	return (sh->cdev->config.devx && sh->config.dv_flow_en &&
-		sh->dev_cap.dest_tir);
+	return sh->cdev->config.devx;
+}
+
+/**
+ * Indicates whether the port uses DevX object operations.
+ *
+ * @param priv
+ *   Pointer to port private data.
+ *
+ * @return
+ *   True if DevX object operations are selected, false otherwise.
+ */
+static inline bool
+mlx5_devx_obj_ops_en(struct mlx5_priv *priv)
+{
+	return priv->obj_ops.devx;
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c
index 4b30a4fade5..abb8e6588f4 100644
--- a/drivers/net/mlx5/mlx5_devx.c
+++ b/drivers/net/mlx5/mlx5_devx.c
@@ -58,7 +58,7 @@ mlx5_devx_extq_port_validate(uint16_t port_id)
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	if (!mlx5_devx_obj_ops_en(priv->sh)) {
+	if (!mlx5_devx_obj_ops_en(priv)) {
 		DRV_LOG(ERR,
 			"Port %u external queue isn't supported by Verbs API.",
 			port_id);
@@ -1754,4 +1754,5 @@ struct mlx5_obj_ops mlx5_devx_obj_ops = {
 	.txq_obj_release = mlx5_txq_devx_obj_release,
 	.lb_dummy_queue_create = NULL,
 	.lb_dummy_queue_release = NULL,
+	.devx = true,
 };
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 306c1cd734e..26ee58d1641 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -925,7 +925,7 @@ mlx5_hairpin_cap_get(struct rte_eth_dev *dev, struct rte_eth_hairpin_cap *cap)
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hca_attr *hca_attr;
 
-	if (!mlx5_devx_obj_ops_en(priv->sh)) {
+	if (!mlx5_devx_obj_ops_en(priv)) {
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index c5f493117ba..ef71b59fe9c 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1364,7 +1364,7 @@ mlx5_dev_start(struct rte_eth_dev *dev)
 		SAVE_RTE_ERRNO_AND_STOP(ret, dev);
 		goto error;
 	}
-	if (mlx5_devx_obj_ops_en(priv->sh) &&
+	if (mlx5_devx_obj_ops_en(priv) &&
 	    priv->obj_ops.lb_dummy_queue_create) {
 		ret = priv->obj_ops.lb_dummy_queue_create(dev);
 		if (ret) {
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index f9b254c6c5d..aa43cb16127 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -695,19 +695,17 @@ mlx5_txq_obj_verify(struct rte_eth_dev *dev)
  *
  * @param txq_ctrl
  *   Pointer to Tx queue control structure.
- * @param devx
- *   If the calculation is used for Devx queue.
  *
  * @return
  *   The number of WQEBB.
  */
 static int
-txq_calc_wqebb_cnt(struct mlx5_txq_ctrl *txq_ctrl, bool devx)
+txq_calc_wqebb_cnt(struct mlx5_txq_ctrl *txq_ctrl)
 {
 	unsigned int wqe_size;
 	const unsigned int desc = 1 << txq_ctrl->txq.elts_n;
 
-	if (devx) {
+	if (mlx5_devx_obj_ops_en(txq_ctrl->priv)) {
 		wqe_size = txq_ctrl->txq.tso_en ?
 			   RTE_ALIGN(txq_ctrl->max_tso_header, MLX5_WSEG_SIZE) : 0;
 		wqe_size += MLX5_WQE_CSEG_SIZE +
@@ -1171,7 +1169,7 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	tmpl->txq.idx = idx;
 	txq_set_params(tmpl);
 	txq_adjust_params(tmpl);
-	wqebb_cnt = txq_calc_wqebb_cnt(tmpl, !!mlx5_devx_obj_ops_en(priv->sh));
+	wqebb_cnt = txq_calc_wqebb_cnt(tmpl);
 	max_wqe = mlx5_dev_get_max_wq_size(priv->sh);
 	if (wqebb_cnt > max_wqe) {
 		DRV_LOG(ERR,
@@ -1721,7 +1719,7 @@ int mlx5_map_aggr_tx_affinity(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 	struct mlx5_priv *priv;
 
 	priv = dev->data->dev_private;
-	if (!mlx5_devx_obj_ops_en(priv->sh)) {
+	if (!mlx5_devx_obj_ops_en(priv)) {
 		DRV_LOG(ERR, "Tx affinity mapping isn't supported by Verbs API.");
 		rte_errno = ENOTSUP;
 		return -rte_errno;
-- 
2.21.0

[PATCH 2/2] net/mlx5: skip DevX Tx memory allocation for Verbs

[Maayan Kashani]

From: Banoth Saikumar <banoth.saikumar@oracle.com>

Previously, the mlx5 PMD attempted to allocate consecutive Tx memory
using DevX without checking whether DevX object operations are used.
This led to allocation failures on legacy or unsupported NICs.

Skip that allocation when the port uses Verbs object operations and let
the Verbs path handle memory allocation and registration.

Fixes: bbfab2eb2528 ("net/mlx5: allocate and release unique resources for Tx queues")
Cc: stable@dpdk.org

Signed-off-by: Banoth Saikumar <banoth.saikumar@oracle.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
 .mailmap                        | 1 +
 drivers/net/mlx5/mlx5_trigger.c | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.mailmap b/.mailmap
index 6d9e262cfb3..d5cfb05b37e 100644
--- a/.mailmap
+++ b/.mailmap
@@ -190,6 +190,7 @@ Ayuj Verma <ayverma@marvell.com> <ayuj.verma@caviumnetworks.com>
 Balakrishna Bhamidipati <bbhamidipati@marvell.com>
 Balasubramanian Manoharan <balasubramanian.manoharan@caviumnetworks.com>
 Balazs Nemeth <bnemeth@redhat.com> <balazs.nemeth@intel.com>
+Banoth Saikumar <banoth.saikumar@oracle.com>
 Bao-Long Tran <longtb5@viettel.com.vn>
 Baoyuan Li <updoing@sina.com>
 Bar Neuman <bneuman@nvidia.com>
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index ef71b59fe9c..1dbde509d26 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1201,7 +1201,7 @@ static int mlx5_dev_allocate_consec_tx_mem(struct rte_eth_dev *dev)
 	void *umem_buf = NULL;
 
 	/* Legacy per queue allocation, do nothing here. */
-	if (priv->sh->config.txq_mem_algn == 0)
+	if (priv->sh->config.txq_mem_algn == 0 || !mlx5_devx_obj_ops_en(priv))
 		return 0;
 	alignment = (size_t)1 << priv->sh->config.txq_mem_algn;
 	total_size = priv->consec_tx_mem.sq_total_size + priv->consec_tx_mem.cq_total_size;
-- 
2.21.0