[PATCH net-next 1/3] net/mlx5: initialize doorbell dma pools

[Tariq Toukan <tariqt@nvidia.com>]

From: Nimrod Oren <noren@nvidia.com>

Add per-node doorbell dma pool creation and cleanup to mdev lifecycle.

Signed-off-by: Nimrod Oren <noren@nvidia.com>
Reviewed-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 .../net/ethernet/mellanox/mlx5/core/alloc.c   | 37 +++++++++++++++++++
 .../net/ethernet/mellanox/mlx5/core/main.c    |  7 ++++
 .../ethernet/mellanox/mlx5/core/mlx5_core.h   |  2 +
 include/linux/mlx5/driver.h                   |  2 +
 4 files changed, 48 insertions(+)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
index 4fe9d7d4f143..3c9938068c56 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
@@ -444,6 +444,43 @@ void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf)
 }
 EXPORT_SYMBOL_GPL(mlx5_frag_buf_free);
 
+void mlx5_db_pools_cleanup(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	int node;
+
+	for_each_node_state(node, N_POSSIBLE)
+		if (priv->db_node_pools[node])
+			mlx5_dma_pool_destroy(priv->db_node_pools[node]);
+
+	kfree(priv->db_node_pools);
+	priv->db_node_pools = NULL;
+}
+
+int mlx5_db_pools_init(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	int node;
+
+	priv->db_node_pools = kzalloc_objs(*priv->db_node_pools, nr_node_ids);
+	if (!priv->db_node_pools)
+		return -ENOMEM;
+
+	for_each_node_state(node, N_POSSIBLE) {
+		struct mlx5_dma_pool *pool;
+
+		pool = mlx5_dma_pool_create(dev, node,
+					    order_base_2(cache_line_size()));
+		if (!pool) {
+			mlx5_db_pools_cleanup(dev);
+			return -ENOMEM;
+		}
+		priv->db_node_pools[node] = pool;
+	}
+
+	return 0;
+}
+
 static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev,
 						 int node)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 643b4aac2033..b3cb090b5677 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -1830,6 +1830,10 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)
 	if (err)
 		goto err_frag_buf_pools_init;
 
+	err = mlx5_db_pools_init(dev);
+	if (err)
+		goto err_db_pools_init;
+
 	INIT_LIST_HEAD(&priv->traps);
 
 	err = mlx5_cmd_init(dev);
@@ -1891,6 +1895,8 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)
 err_timeout_init:
 	mlx5_cmd_cleanup(dev);
 err_cmd_init:
+	mlx5_db_pools_cleanup(dev);
+err_db_pools_init:
 	mlx5_frag_buf_pools_cleanup(dev);
 err_frag_buf_pools_init:
 	debugfs_remove(dev->priv.dbg.dbg_root);
@@ -1917,6 +1923,7 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev)
 	mlx5_health_cleanup(dev);
 	mlx5_tout_cleanup(dev);
 	mlx5_cmd_cleanup(dev);
+	mlx5_db_pools_cleanup(dev);
 	mlx5_frag_buf_pools_cleanup(dev);
 	debugfs_remove_recursive(dev->priv.dbg.dbg_root);
 	mutex_destroy(&priv->pgdir_mutex);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index 09e669f83dba..d6713a2ce676 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -438,6 +438,8 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx);
 void mlx5_mdev_uninit(struct mlx5_core_dev *dev);
 int mlx5_frag_buf_pools_init(struct mlx5_core_dev *dev);
 void mlx5_frag_buf_pools_cleanup(struct mlx5_core_dev *dev);
+int mlx5_db_pools_init(struct mlx5_core_dev *dev);
+void mlx5_db_pools_cleanup(struct mlx5_core_dev *dev);
 int mlx5_init_one(struct mlx5_core_dev *dev);
 int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev);
 void mlx5_uninit_one(struct mlx5_core_dev *dev);
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index b1871c0821d0..4246d6d904ba 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -569,6 +569,7 @@ enum mlx5_page_mgt_mode {
 };
 
 struct mlx5_frag_buf_node_pools;
+struct mlx5_dma_pool;
 struct mlx5_ft_pool;
 struct mlx5_priv {
 	/* IRQ table valid only for real pci devices PF or VF */
@@ -602,6 +603,7 @@ struct mlx5_priv {
 	struct list_head        pgdir_list;
 
 	struct mlx5_frag_buf_node_pools **frag_buf_node_pools;
+	struct mlx5_dma_pool **db_node_pools;
 	/* end: alloc stuff */
 
 	struct mlx5_adev       **adev;
-- 
2.44.0