[dpdk-dev] [PATCH 4/6] common/mlx5: fix doorbell mapping configuration

[<michaelba@nvidia.com>]

From: Michael Baum <michaelba@oss.nvidia.com>

UAR mapping type can be affected by the devarg tx_db_nc, which can cause
setting the environment variable MLX5_SHUT_UP_BF.
So, the MLX5_SHUT_UP_BF value and the UAR mapping parameter affect the
UAR cache mode.

Wrongly, the devarg was considered for the MLX5_SHUT_UP_BF but not for
the UAR mapping parameter in all the drivers except the net.

Take the tx_db_nc devarg into account for all the drivers.

Fixes: ca1418ce3910 ("common/mlx5: share device context object")
Cc: stable@dpdk.org

Signed-off-by: Michael Baum <michaelba@oss.nvidia.com>
Reviewed-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.c     | 52 ++++++++++++++-------------
 drivers/common/mlx5/mlx5_common.h     |  5 +--
 drivers/compress/mlx5/mlx5_compress.c |  2 +-
 drivers/crypto/mlx5/mlx5_crypto.c     |  2 +-
 drivers/regex/mlx5/mlx5_regex.c       |  2 +-
 drivers/vdpa/mlx5/mlx5_vdpa_event.c   |  2 +-
 6 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index 7f92e3b2cc..7bdc550b36 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -934,30 +934,25 @@ RTE_INIT_PRIO(mlx5_is_haswell_broadwell_cpu, LOG)
 
 /**
  * Allocate the User Access Region with DevX on specified device.
+ * This routine handles the following UAR allocation issues:
  *
- * @param [in] ctx
- *   Infiniband device context to perform allocation on.
- * @param [in] mapping
- *   MLX5DV_UAR_ALLOC_TYPE_BF - allocate as cached memory with write-combining
- *				attributes (if supported by the host), the
- *				writes to the UAR registers must be followed
- *				by write memory barrier.
- *   MLX5DV_UAR_ALLOC_TYPE_NC - allocate as non-cached memory, all writes are
- *				promoted to the registers immediately, no
- *				memory barriers needed.
- *   mapping < 0 - the first attempt is performed with MLX5DV_UAR_ALLOC_TYPE_NC,
- *		   if this fails the next attempt with MLX5DV_UAR_ALLOC_TYPE_BF
- *		   is performed. The drivers specifying negative values should
- *		   always provide the write memory barrier operation after UAR
- *		   register writings.
- * If there is no definitions for the MLX5DV_UAR_ALLOC_TYPE_xx (older rdma
- * library headers), the caller can specify 0.
+ *  - tries to allocate the UAR with the most appropriate memory mapping
+ *    type from the ones supported by the host.
+ *
+ *  - tries to allocate the UAR with non-NULL base address OFED 5.0.x and
+ *    Upstream rdma_core before v29 returned the NULL as UAR base address
+ *    if UAR was not the first object in the UAR page.
+ *    It caused the PMD failure and we should try to get another UAR till
+ *    we get the first one with non-NULL base address returned.
+ *
+ * @param [in] cdev
+ *   Pointer to mlx5 device structure to perform allocation on its context.
  *
  * @return
  *   UAR object pointer on success, NULL otherwise and rte_errno is set.
  */
 void *
-mlx5_devx_alloc_uar(void *ctx, int mapping)
+mlx5_devx_alloc_uar(struct mlx5_common_device *cdev)
 {
 	void *uar;
 	uint32_t retry, uar_mapping;
@@ -966,26 +961,35 @@ mlx5_devx_alloc_uar(void *ctx, int mapping)
 	for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) {
 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC
 		/* Control the mapping type according to the settings. */
-		uar_mapping = (mapping < 0) ?
-			      MLX5DV_UAR_ALLOC_TYPE_NC : mapping;
+		uar_mapping = (cdev->config.dbnc == MLX5_TXDB_NCACHED) ?
+			    MLX5DV_UAR_ALLOC_TYPE_NC : MLX5DV_UAR_ALLOC_TYPE_BF;
 #else
 		/*
 		 * It seems we have no way to control the memory mapping type
 		 * for the UAR, the default "Write-Combining" type is supposed.
 		 */
 		uar_mapping = 0;
-		RTE_SET_USED(mapping);
 #endif
-		uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping);
+		uar = mlx5_glue->devx_alloc_uar(cdev->ctx, uar_mapping);
 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC
-		if (!uar && mapping < 0) {
+		if (!uar && uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) {
+			/*
+			 * In some environments like virtual machine the
+			 * Write Combining mapped might be not supported and
+			 * UAR allocation fails. We tried "Non-Cached" mapping
+			 * for the case.
+			 */
+			DRV_LOG(DEBUG, "Failed to allocate DevX UAR (BF)");
+			uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC;
+			uar = mlx5_glue->devx_alloc_uar(cdev->ctx, uar_mapping);
+		} else if (!uar && uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) {
 			/*
 			 * If Verbs/kernel does not support "Non-Cached"
 			 * try the "Write-Combining".
 			 */
 			DRV_LOG(DEBUG, "Failed to allocate DevX UAR (NC)");
 			uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF;
-			uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping);
+			uar = mlx5_glue->devx_alloc_uar(cdev->ctx, uar_mapping);
 		}
 #endif
 		if (!uar) {
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 744c6a72b3..7febae9cdf 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -284,8 +284,6 @@ __rte_internal
 void mlx5_translate_port_name(const char *port_name_in,
 			      struct mlx5_switch_info *port_info_out);
 void mlx5_glue_constructor(void);
-__rte_internal
-void *mlx5_devx_alloc_uar(void *ctx, int mapping);
 extern uint8_t haswell_broadwell_cpu;
 
 __rte_internal
@@ -417,6 +415,9 @@ void
 mlx5_dev_mempool_unregister(struct mlx5_common_device *cdev,
 			    struct rte_mempool *mp);
 
+__rte_internal
+void *mlx5_devx_alloc_uar(struct mlx5_common_device *cdev);
+
 /* mlx5_common_mr.c */
 
 __rte_internal
diff --git a/drivers/compress/mlx5/mlx5_compress.c b/drivers/compress/mlx5/mlx5_compress.c
index c4081c5f7d..df60b05ab3 100644
--- a/drivers/compress/mlx5/mlx5_compress.c
+++ b/drivers/compress/mlx5/mlx5_compress.c
@@ -690,7 +690,7 @@ mlx5_compress_uar_release(struct mlx5_compress_priv *priv)
 static int
 mlx5_compress_uar_prepare(struct mlx5_compress_priv *priv)
 {
-	priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
+	priv->uar = mlx5_devx_alloc_uar(priv->cdev);
 	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
 	    NULL) {
 		rte_errno = errno;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index f9fd0d498e..33d797a6a0 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -731,7 +731,7 @@ mlx5_crypto_uar_release(struct mlx5_crypto_priv *priv)
 static int
 mlx5_crypto_uar_prepare(struct mlx5_crypto_priv *priv)
 {
-	priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
+	priv->uar = mlx5_devx_alloc_uar(priv->cdev);
 	if (priv->uar)
 		priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
 	if (priv->uar == NULL || priv->uar_addr == NULL) {
diff --git a/drivers/regex/mlx5/mlx5_regex.c b/drivers/regex/mlx5/mlx5_regex.c
index b8a513e1fa..d632252794 100644
--- a/drivers/regex/mlx5/mlx5_regex.c
+++ b/drivers/regex/mlx5/mlx5_regex.c
@@ -138,7 +138,7 @@ mlx5_regex_dev_probe(struct mlx5_common_device *cdev)
 	 * registers writings, it is safe to allocate UAR with any
 	 * memory mapping type.
 	 */
-	priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
+	priv->uar = mlx5_devx_alloc_uar(priv->cdev);
 	if (!priv->uar) {
 		DRV_LOG(ERR, "can't allocate uar.");
 		rte_errno = ENOMEM;
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_event.c b/drivers/vdpa/mlx5/mlx5_vdpa_event.c
index 042d22777f..21738bdfff 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_event.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_event.c
@@ -61,7 +61,7 @@ mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv)
 	 * registers writings, it is safe to allocate UAR with any
 	 * memory mapping type.
 	 */
-	priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
+	priv->uar = mlx5_devx_alloc_uar(priv->cdev);
 	if (!priv->uar) {
 		rte_errno = errno;
 		DRV_LOG(ERR, "Failed to allocate UAR.");
-- 
2.25.1