From: Saeed Mahameed <saeed@kernel.org>
To: "David S. Miller" <davem@davemloft.net>,
Jakub Kicinski <kuba@kernel.org>
Cc: netdev@vger.kernel.org, Shay Drory <shayd@nvidia.com>,
Moshe Shemesh <moshe@nvidia.com>, Parav Pandit <parav@nvidia.com>,
Saeed Mahameed <saeedm@nvidia.com>
Subject: [net-next 10/14] net/mlx5: Let user configure io_eq_size param
Date: Mon, 25 Oct 2021 13:54:27 -0700 [thread overview]
Message-ID: <20211025205431.365080-11-saeed@kernel.org> (raw)
In-Reply-To: <20211025205431.365080-1-saeed@kernel.org>
From: Shay Drory <shayd@nvidia.com>
Currently, each I/O EQ is taking 128KB of memory. This size
is not needed in all use cases, and is critical with large scale.
Hence, allow user to configure the size of I/O EQs.
For example, to reduce I/O EQ size to 64, execute:
$ devlink resource set pci/0000:00:0b.0 path /io_eq_size/ size 64
$ devlink dev reload pci/0000:00:0b.0
Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Moshe Shemesh <moshe@nvidia.com>
Reviewed-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
---
Documentation/networking/devlink/mlx5.rst | 12 ++++
.../net/ethernet/mellanox/mlx5/core/Makefile | 2 +-
.../net/ethernet/mellanox/mlx5/core/devlink.h | 11 ++++
.../ethernet/mellanox/mlx5/core/devlink_res.c | 56 +++++++++++++++++++
drivers/net/ethernet/mellanox/mlx5/core/eq.c | 3 +-
.../net/ethernet/mellanox/mlx5/core/main.c | 3 +
include/linux/mlx5/driver.h | 4 --
7 files changed, 85 insertions(+), 6 deletions(-)
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/devlink_res.c
diff --git a/Documentation/networking/devlink/mlx5.rst b/Documentation/networking/devlink/mlx5.rst
index 4e4b97f7971a..4e6020570292 100644
--- a/Documentation/networking/devlink/mlx5.rst
+++ b/Documentation/networking/devlink/mlx5.rst
@@ -46,6 +46,18 @@ parameters.
The ``mlx5`` driver supports reloading via ``DEVLINK_CMD_RELOAD``
+Resources
+=========
+
+.. list-table:: Driver-specific resources implemented
+ :widths: 5 5 5 85
+
+ * - Name
+ - Description
+ * - ``comp_eq_size``
+ - Control the size of I/O completion EQs.
+ * The default value is 1024, and the range is between 64 and 4096.
+
Info versions
=============
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index bdb271b604d9..79c15ee62cde 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -16,7 +16,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \
fs_counters.o fs_ft_pool.o rl.o lag/lag.o dev.o events.o wq.o lib/gid.o \
lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \
- diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o \
+ diag/fw_tracer.o diag/crdump.o devlink.o devlink_res.o diag/rsc_dump.o \
fw_reset.o qos.o lib/tout.o
#
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h
index 30bf4882779b..4192f23b1446 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h
@@ -6,6 +6,13 @@
#include <net/devlink.h>
+enum mlx5_devlink_resource_id {
+ MLX5_DL_RES_COMP_EQ = 1,
+
+ __MLX5_ID_RES_MAX,
+ MLX5_ID_RES_MAX = __MLX5_ID_RES_MAX - 1,
+};
+
enum mlx5_devlink_param_id {
MLX5_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
MLX5_DEVLINK_PARAM_ID_FLOW_STEERING_MODE,
@@ -31,6 +38,10 @@ int mlx5_devlink_trap_get_num_active(struct mlx5_core_dev *dev);
int mlx5_devlink_traps_get_action(struct mlx5_core_dev *dev, int trap_id,
enum devlink_trap_action *action);
+void mlx5_devlink_res_register(struct mlx5_core_dev *dev);
+void mlx5_devlink_res_unregister(struct mlx5_core_dev *dev);
+size_t mlx5_devlink_res_size(struct mlx5_core_dev *dev, enum mlx5_devlink_resource_id id);
+
struct devlink *mlx5_devlink_alloc(struct device *dev);
void mlx5_devlink_free(struct devlink *devlink);
int mlx5_devlink_register(struct devlink *devlink);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink_res.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink_res.c
new file mode 100644
index 000000000000..3beedfb8534a
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink_res.c
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */
+
+#include "devlink.h"
+#include "mlx5_core.h"
+
+enum {
+ MLX5_EQ_MIN_SIZE = 64,
+ MLX5_EQ_MAX_SIZE = 4096,
+ MLX5_COMP_EQ_SIZE = 1024,
+};
+
+static int comp_eq_res_register(struct mlx5_core_dev *dev)
+{
+ struct devlink_resource_size_params comp_eq_size;
+ struct devlink *devlink = priv_to_devlink(dev);
+
+ devlink_resource_size_params_init(&comp_eq_size, MLX5_EQ_MIN_SIZE,
+ MLX5_EQ_MAX_SIZE, 1, DEVLINK_RESOURCE_UNIT_ENTRY);
+ return devlink_resource_register(devlink, "io_eq_size", MLX5_COMP_EQ_SIZE,
+ MLX5_DL_RES_COMP_EQ,
+ DEVLINK_RESOURCE_ID_PARENT_TOP,
+ &comp_eq_size);
+}
+
+void mlx5_devlink_res_register(struct mlx5_core_dev *dev)
+{
+ int err;
+
+ err = comp_eq_res_register(dev);
+ if (err)
+ mlx5_core_err(dev, "Failed to register resources, err = %d\n", err);
+}
+
+void mlx5_devlink_res_unregister(struct mlx5_core_dev *dev)
+{
+ devlink_resources_unregister(priv_to_devlink(dev), NULL);
+}
+
+static const size_t default_vals[MLX5_ID_RES_MAX + 1] = {
+ [MLX5_DL_RES_COMP_EQ] = MLX5_COMP_EQ_SIZE,
+};
+
+size_t mlx5_devlink_res_size(struct mlx5_core_dev *dev, enum mlx5_devlink_resource_id id)
+{
+ struct devlink *devlink = priv_to_devlink(dev);
+ u64 size;
+ int err;
+
+ err = devlink_resource_size_get(devlink, id, &size);
+ if (!err)
+ return size;
+ mlx5_core_err(dev, "Failed to get param. using default. err = %d, id = %u\n",
+ err, id);
+ return default_vals[id];
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 792e0d6aa861..4dda6e2a4cbc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -19,6 +19,7 @@
#include "lib/clock.h"
#include "diag/fw_tracer.h"
#include "mlx5_irq.h"
+#include "devlink.h"
enum {
MLX5_EQE_OWNER_INIT_VAL = 0x1,
@@ -807,7 +808,7 @@ static int create_comp_eqs(struct mlx5_core_dev *dev)
INIT_LIST_HEAD(&table->comp_eqs_list);
ncomp_eqs = table->num_comp_eqs;
- nent = MLX5_COMP_EQ_SIZE;
+ nent = mlx5_devlink_res_size(dev, MLX5_DL_RES_COMP_EQ);
for (i = 0; i < ncomp_eqs; i++) {
struct mlx5_eq_param param = {};
int vecidx = i;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index f8446395163a..96fdbc0c87bf 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -922,6 +922,8 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)
dev->hv_vhca = mlx5_hv_vhca_create(dev);
dev->rsc_dump = mlx5_rsc_dump_create(dev);
+ mlx5_devlink_res_register(dev);
+
return 0;
err_sf_table_cleanup:
@@ -957,6 +959,7 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)
static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
{
+ mlx5_devlink_res_unregister(dev);
mlx5_rsc_dump_destroy(dev);
mlx5_hv_vhca_destroy(dev->hv_vhca);
mlx5_fw_tracer_destroy(dev->tracer);
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index f617dfbcd9fd..47c07f95bbe1 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -797,10 +797,6 @@ struct mlx5_db {
int index;
};
-enum {
- MLX5_COMP_EQ_SIZE = 1024,
-};
-
enum {
MLX5_PTYS_IB = 1 << 0,
MLX5_PTYS_EN = 1 << 2,
--
2.31.1
next prev parent reply other threads:[~2021-10-25 20:54 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-10-25 20:54 [pull request][net-next 00/14] mlx5 updates 2021-10-25 Saeed Mahameed
2021-10-25 20:54 ` [net-next 01/14] net/mlx5e: don't write directly to netdev->dev_addr Saeed Mahameed
2021-10-26 12:30 ` patchwork-bot+netdevbpf
2021-10-25 20:54 ` [net-next 02/14] net/mlx5: Remove unnecessary checks for slow path flag Saeed Mahameed
2021-10-25 20:54 ` [net-next 03/14] net/mlx5: Fix unused function warning of mlx5i_flow_type_mask Saeed Mahameed
2021-10-25 20:54 ` [net-next 04/14] net/mlx5: Reduce flow counters bulk query buffer size for SFs Saeed Mahameed
2021-10-25 20:54 ` [net-next 05/14] net/mlx5: Extend health buffer dump Saeed Mahameed
2021-10-25 20:54 ` [net-next 06/14] net/mlx5: Print health buffer by log level Saeed Mahameed
2021-10-25 20:54 ` [net-next 07/14] net/mlx5: Add periodic update of host time to firmware Saeed Mahameed
2021-10-25 20:54 ` [net-next 08/14] net/mlx5: Bridge, extract code to lookup and del/notify entry Saeed Mahameed
2021-10-25 20:54 ` [net-next 09/14] net/mlx5: Bridge, support replacing existing FDB entry Saeed Mahameed
2021-10-25 20:54 ` Saeed Mahameed [this message]
2021-10-26 15:05 ` [net-next 10/14] net/mlx5: Let user configure io_eq_size param Jakub Kicinski
2021-10-26 15:54 ` Saeed Mahameed
2021-10-26 17:16 ` Jakub Kicinski
2021-10-26 18:01 ` Saeed Mahameed
2021-10-27 6:16 ` Gal Pressman
2021-10-25 20:54 ` [net-next 11/14] net/mlx5: Let user configure event_eq_size param Saeed Mahameed
2021-10-25 20:54 ` [net-next 12/14] net/mlx5: Let user configure max_macs param Saeed Mahameed
2021-10-25 20:54 ` [net-next 13/14] net/mlx5: SF, Add SF trace points Saeed Mahameed
2021-10-25 20:54 ` [net-next 14/14] net/mlx5: SF_DEV Add SF device " Saeed Mahameed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211025205431.365080-11-saeed@kernel.org \
--to=saeed@kernel.org \
--cc=davem@davemloft.net \
--cc=kuba@kernel.org \
--cc=moshe@nvidia.com \
--cc=netdev@vger.kernel.org \
--cc=parav@nvidia.com \
--cc=saeedm@nvidia.com \
--cc=shayd@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.