From: Saeed Mahameed <saeed@kernel.org>
To: "David S. Miller" <davem@davemloft.net>,
Jakub Kicinski <kuba@kernel.org>
Cc: netdev@vger.kernel.org, Dmytro Linkin <dlinkin@nvidia.com>,
Parav Pandit <parav@nvidia.com>, Mark Bloch <mbloch@nvidia.com>,
Saeed Mahameed <saeedm@nvidia.com>
Subject: [net-next v0 14/15] net/mlx5: E-switch, Enable vport QoS on demand
Date: Tue, 16 Nov 2021 20:33:56 -0800 [thread overview]
Message-ID: <20211117043357.345072-15-saeed@kernel.org> (raw)
In-Reply-To: <20211117043357.345072-1-saeed@kernel.org>
From: Dmytro Linkin <dlinkin@nvidia.com>
Vports' QoS is not commonly used but consume SW/HW resources, which
becomes an issue on BlueField SoC systems.
Don't enable QoS on vports by default on eswitch mode change and enable
when it's going to be used by one of the top level users:
- configuring TC matchall filter with police action;
- setting rate with legacy NDO API;
- calling devlink ops->rate_leaf_*() callbacks.
Disable vport QoS on vport cleanup.
Signed-off-by: Dmytro Linkin <dlinkin@nvidia.com>
Reviewed-by: Parav Pandit <parav@nvidia.com>
Reviewed-by: Mark Bloch <mbloch@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
---
.../ethernet/mellanox/mlx5/core/esw/legacy.c | 4 +-
.../net/ethernet/mellanox/mlx5/core/esw/qos.c | 82 +++++++++++++------
.../net/ethernet/mellanox/mlx5/core/esw/qos.h | 12 +--
.../net/ethernet/mellanox/mlx5/core/eswitch.c | 9 +-
4 files changed, 64 insertions(+), 43 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c
index df277a6cddc0..2b52f7c09152 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c
@@ -522,9 +522,7 @@ int mlx5_eswitch_set_vport_rate(struct mlx5_eswitch *esw, u16 vport,
return PTR_ERR(evport);
mutex_lock(&esw->state_lock);
- err = mlx5_esw_qos_set_vport_min_rate(esw, evport, min_rate, NULL);
- if (!err)
- err = mlx5_esw_qos_set_vport_max_rate(esw, evport, max_rate, NULL);
+ err = mlx5_esw_qos_set_vport_rate(esw, evport, max_rate, min_rate);
mutex_unlock(&esw->state_lock);
return err;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
index c6cc67cb4f6a..304abc293086 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
@@ -204,10 +204,8 @@ static int esw_qos_normalize_groups_min_rate(struct mlx5_eswitch *esw, u32 divid
return 0;
}
-int mlx5_esw_qos_set_vport_min_rate(struct mlx5_eswitch *esw,
- struct mlx5_vport *evport,
- u32 min_rate,
- struct netlink_ext_ack *extack)
+static int esw_qos_set_vport_min_rate(struct mlx5_eswitch *esw, struct mlx5_vport *evport,
+ u32 min_rate, struct netlink_ext_ack *extack)
{
u32 fw_max_bw_share, previous_min_rate;
bool min_rate_supported;
@@ -231,10 +229,8 @@ int mlx5_esw_qos_set_vport_min_rate(struct mlx5_eswitch *esw,
return err;
}
-int mlx5_esw_qos_set_vport_max_rate(struct mlx5_eswitch *esw,
- struct mlx5_vport *evport,
- u32 max_rate,
- struct netlink_ext_ack *extack)
+static int esw_qos_set_vport_max_rate(struct mlx5_eswitch *esw, struct mlx5_vport *evport,
+ u32 max_rate, struct netlink_ext_ack *extack)
{
u32 act_max_rate = max_rate;
bool max_rate_supported;
@@ -605,8 +601,8 @@ void mlx5_esw_qos_destroy(struct mlx5_eswitch *esw)
mutex_unlock(&esw->state_lock);
}
-int mlx5_esw_qos_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
- u32 max_rate, u32 bw_share)
+static int esw_qos_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
+ u32 max_rate, u32 bw_share)
{
int err;
@@ -615,7 +611,7 @@ int mlx5_esw_qos_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport
return 0;
if (vport->qos.enabled)
- return -EEXIST;
+ return 0;
vport->qos.group = esw->qos.group0;
@@ -645,31 +641,55 @@ void mlx5_esw_qos_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vpo
esw_warn(esw->dev, "E-Switch destroy TSAR vport element failed (vport=%d,err=%d)\n",
vport->vport, err);
- vport->qos.enabled = false;
+ memset(&vport->qos, 0, sizeof(vport->qos));
trace_mlx5_esw_vport_qos_destroy(vport);
}
+int mlx5_esw_qos_set_vport_rate(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
+ u32 min_rate, u32 max_rate)
+{
+ int err;
+
+ lockdep_assert_held(&esw->state_lock);
+ err = esw_qos_vport_enable(esw, vport, 0, 0);
+ if (err)
+ return err;
+
+ err = esw_qos_set_vport_min_rate(esw, vport, min_rate, NULL);
+ if (!err)
+ err = esw_qos_set_vport_max_rate(esw, vport, max_rate, NULL);
+
+ return err;
+}
+
int mlx5_esw_qos_modify_vport_rate(struct mlx5_eswitch *esw, u16 vport_num, u32 rate_mbps)
{
u32 ctx[MLX5_ST_SZ_DW(scheduling_context)] = {};
struct mlx5_vport *vport;
u32 bitmask;
+ int err;
vport = mlx5_eswitch_get_vport(esw, vport_num);
if (IS_ERR(vport))
return PTR_ERR(vport);
- if (!vport->qos.enabled)
- return -EOPNOTSUPP;
-
- MLX5_SET(scheduling_context, ctx, max_average_bw, rate_mbps);
- bitmask = MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_MAX_AVERAGE_BW;
+ mutex_lock(&esw->state_lock);
+ if (!vport->qos.enabled) {
+ /* Eswitch QoS wasn't enabled yet. Enable it and vport QoS. */
+ err = esw_qos_vport_enable(esw, vport, rate_mbps, vport->qos.bw_share);
+ } else {
+ MLX5_SET(scheduling_context, ctx, max_average_bw, rate_mbps);
+
+ bitmask = MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_MAX_AVERAGE_BW;
+ err = mlx5_modify_scheduling_element_cmd(esw->dev,
+ SCHEDULING_HIERARCHY_E_SWITCH,
+ ctx,
+ vport->qos.esw_tsar_ix,
+ bitmask);
+ }
+ mutex_unlock(&esw->state_lock);
- return mlx5_modify_scheduling_element_cmd(esw->dev,
- SCHEDULING_HIERARCHY_E_SWITCH,
- ctx,
- vport->qos.esw_tsar_ix,
- bitmask);
+ return err;
}
#define MLX5_LINKSPEED_UNIT 125000 /* 1Mbps in Bps */
@@ -728,7 +748,12 @@ int mlx5_esw_devlink_rate_leaf_tx_share_set(struct devlink_rate *rate_leaf, void
return err;
mutex_lock(&esw->state_lock);
- err = mlx5_esw_qos_set_vport_min_rate(esw, vport, tx_share, extack);
+ err = esw_qos_vport_enable(esw, vport, 0, 0);
+ if (err)
+ goto unlock;
+
+ err = esw_qos_set_vport_min_rate(esw, vport, tx_share, extack);
+unlock:
mutex_unlock(&esw->state_lock);
return err;
}
@@ -749,7 +774,12 @@ int mlx5_esw_devlink_rate_leaf_tx_max_set(struct devlink_rate *rate_leaf, void *
return err;
mutex_lock(&esw->state_lock);
- err = mlx5_esw_qos_set_vport_max_rate(esw, vport, tx_max, extack);
+ err = esw_qos_vport_enable(esw, vport, 0, 0);
+ if (err)
+ goto unlock;
+
+ err = esw_qos_set_vport_max_rate(esw, vport, tx_max, extack);
+unlock:
mutex_unlock(&esw->state_lock);
return err;
}
@@ -846,7 +876,9 @@ int mlx5_esw_qos_vport_update_group(struct mlx5_eswitch *esw,
int err;
mutex_lock(&esw->state_lock);
- err = esw_qos_vport_update_group(esw, vport, group, extack);
+ err = esw_qos_vport_enable(esw, vport, 0, 0);
+ if (!err)
+ err = esw_qos_vport_update_group(esw, vport, group, extack);
mutex_unlock(&esw->state_lock);
return err;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h
index 28451abe2d2f..91b66c1b9881 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h
@@ -6,18 +6,10 @@
#ifdef CONFIG_MLX5_ESWITCH
-int mlx5_esw_qos_set_vport_min_rate(struct mlx5_eswitch *esw,
- struct mlx5_vport *evport,
- u32 min_rate,
- struct netlink_ext_ack *extack);
-int mlx5_esw_qos_set_vport_max_rate(struct mlx5_eswitch *esw,
- struct mlx5_vport *evport,
- u32 max_rate,
- struct netlink_ext_ack *extack);
+int mlx5_esw_qos_set_vport_rate(struct mlx5_eswitch *esw, struct mlx5_vport *evport,
+ u32 max_rate, u32 min_rate);
void mlx5_esw_qos_create(struct mlx5_eswitch *esw);
void mlx5_esw_qos_destroy(struct mlx5_eswitch *esw);
-int mlx5_esw_qos_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
- u32 max_rate, u32 bw_share);
void mlx5_esw_qos_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vport);
int mlx5_esw_devlink_rate_leaf_tx_share_set(struct devlink_rate *rate_leaf, void *priv,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index ec5b1641d40c..2d188f462028 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -781,9 +781,6 @@ static int esw_vport_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
if (err)
return err;
- /* Attach vport to the eswitch rate limiter */
- mlx5_esw_qos_vport_enable(esw, vport, vport->qos.max_rate, vport->qos.bw_share);
-
if (mlx5_esw_is_manager_vport(esw, vport_num))
return 0;
@@ -1746,8 +1743,10 @@ int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw,
ivi->qos = evport->info.qos;
ivi->spoofchk = evport->info.spoofchk;
ivi->trusted = evport->info.trusted;
- ivi->min_tx_rate = evport->qos.min_rate;
- ivi->max_tx_rate = evport->qos.max_rate;
+ if (evport->qos.enabled) {
+ ivi->min_tx_rate = evport->qos.min_rate;
+ ivi->max_tx_rate = evport->qos.max_rate;
+ }
mutex_unlock(&esw->state_lock);
return 0;
--
2.31.1
next prev parent reply other threads:[~2021-11-17 4:34 UTC|newest]
Thread overview: 17+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-17 4:33 [pull request][net-next v0 00/15] mlx5 updates 2021-11-16 Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 01/15] net/mlx5e: Support ethtool cq mode Saeed Mahameed
2021-11-17 11:10 ` patchwork-bot+netdevbpf
2021-11-17 4:33 ` [net-next v0 02/15] net/mlx5: Fix format-security build warnings Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 03/15] net/mlx5: Avoid printing health buffer when firmware is unavailable Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 04/15] net/mlx5e: Refactor mod header management API Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 05/15] net/mlx5: CT: Allow static allocation of mod headers Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 06/15] net/mlx5: TC, using swap() instead of tmp variable Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 07/15] net/mlx5e: TC, Destroy nic flow counter if exists Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 08/15] net/mlx5e: TC, Move kfree() calls after destroying all resources Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 09/15] net/mlx5e: TC, Move comment about mod header flag to correct place Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 10/15] net/mlx5e: Specify out ifindex when looking up decap route Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 11/15] net/mlx5: E-switch, Remove vport enabled check Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 12/15] net/mlx5: E-switch, Reuse mlx5_eswitch_set_vport_mac Saeed Mahameed
2021-11-17 4:33 ` [net-next v0 13/15] net/mlx5: E-switch, move offloads mode callbacks to offloads file Saeed Mahameed
2021-11-17 4:33 ` Saeed Mahameed [this message]
2021-11-17 4:33 ` [net-next v0 15/15] net/mlx5: E-switch, Create QoS on demand Saeed Mahameed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211117043357.345072-15-saeed@kernel.org \
--to=saeed@kernel.org \
--cc=davem@davemloft.net \
--cc=dlinkin@nvidia.com \
--cc=kuba@kernel.org \
--cc=mbloch@nvidia.com \
--cc=netdev@vger.kernel.org \
--cc=parav@nvidia.com \
--cc=saeedm@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).