From: Saeed Mahameed <saeed@kernel.org>
To: "David S. Miller" <davem@davemloft.net>,
Jakub Kicinski <kuba@kernel.org>
Cc: netdev@vger.kernel.org, Lama Kayal <lkayal@nvidia.com>,
Maxim Mikityanskiy <maximmi@nvidia.com>,
Tariq Toukan <tariqt@nvidia.com>,
Saeed Mahameed <saeedm@nvidia.com>
Subject: [net 10/10] net/mlx5e: Do synchronize_net only once when deactivating channels
Date: Fri, 19 Nov 2021 11:58:13 -0800 [thread overview]
Message-ID: <20211119195813.739586-11-saeed@kernel.org> (raw)
In-Reply-To: <20211119195813.739586-1-saeed@kernel.org>
From: Lama Kayal <lkayal@nvidia.com>
Synchronize_net takes time, and given that it's called multiple times
per channel on deactivation, it accumulates to a significant amount,
which causes timeouts in some applications (for example, when using
bonding with NetworkManager).
This commit fixes the timing issue by restructuring the code, so that
synchronize_net is only called once per deactivation of all channels.
It's entirely valid, because we only need one synchronization point with
NAPI between deactivation and closing the channels.
Fixes: 9c25a22dfb00 ("net/mlx5e: Use synchronize_rcu to sync with NAPI")
Signed-off-by: Lama Kayal <lkayal@nvidia.com>
Signed-off-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
---
.../net/ethernet/mellanox/mlx5/core/en/ptp.c | 26 ++++++----
.../net/ethernet/mellanox/mlx5/core/en/ptp.h | 6 ++-
.../net/ethernet/mellanox/mlx5/core/en_main.c | 52 +++++++++++++------
3 files changed, 55 insertions(+), 29 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
index f190c5437294..3314fcadec5c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
@@ -724,11 +724,14 @@ void mlx5e_ptp_close(struct mlx5e_ptp *c)
kvfree(c);
}
-void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c)
+void mlx5e_ptp_enable_channel(struct mlx5e_ptp *c)
{
- int tc;
-
napi_enable(&c->napi);
+}
+
+void mlx5e_ptp_start_channel(struct mlx5e_ptp *c)
+{
+ int tc;
if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
for (tc = 0; tc < c->num_tc; tc++) {
@@ -742,22 +745,25 @@ void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c)
}
}
-void mlx5e_ptp_deactivate_channel(struct mlx5e_ptp *c)
+void mlx5e_ptp_disable_channel(struct mlx5e_ptp *c)
{
int tc;
if (test_bit(MLX5E_PTP_STATE_RX, c->state))
mlx5e_deactivate_rq(&c->rq);
- if (test_bit(MLX5E_PTP_STATE_TX, c->state)) {
- for (tc = 0; tc < c->num_tc; tc++) {
+ if (test_bit(MLX5E_PTP_STATE_TX, c->state))
+ for (tc = 0; tc < c->num_tc; tc++)
mlx5e_disable_txqsq(&c->ptpsq[tc].txqsq);
+}
+
+void mlx5e_ptp_stop_channel(struct mlx5e_ptp *c)
+{
+ int tc;
- /* Sync with NAPI to prevent netif_tx_wake_queue. */
- synchronize_net();
+ if (test_bit(MLX5E_PTP_STATE_TX, c->state))
+ for (tc = 0; tc < c->num_tc; tc++)
mlx5e_stop_txqsq(&c->ptpsq[tc].txqsq);
- }
- }
napi_disable(&c->napi);
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h
index a71a32e00ebb..e96d2af682f9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h
@@ -70,8 +70,10 @@ static inline bool mlx5e_use_ptpsq(struct sk_buff *skb)
int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params,
u8 lag_port, struct mlx5e_ptp **cp);
void mlx5e_ptp_close(struct mlx5e_ptp *c);
-void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c);
-void mlx5e_ptp_deactivate_channel(struct mlx5e_ptp *c);
+void mlx5e_ptp_enable_channel(struct mlx5e_ptp *c);
+void mlx5e_ptp_start_channel(struct mlx5e_ptp *c);
+void mlx5e_ptp_disable_channel(struct mlx5e_ptp *c);
+void mlx5e_ptp_stop_channel(struct mlx5e_ptp *c);
int mlx5e_ptp_get_rqn(struct mlx5e_ptp *c, u32 *rqn);
int mlx5e_ptp_alloc_rx_fs(struct mlx5e_priv *priv);
void mlx5e_ptp_free_rx_fs(struct mlx5e_priv *priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 1fb0d9b70301..f77d76728b46 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -2251,11 +2251,14 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
return err;
}
-static void mlx5e_activate_channel(struct mlx5e_channel *c)
+static void mlx5e_enable_channel(struct mlx5e_channel *c)
{
- int tc;
-
napi_enable(&c->napi);
+}
+
+static void mlx5e_start_channel(struct mlx5e_channel *c)
+{
+ int tc;
for (tc = 0; tc < c->num_tc; tc++) {
mlx5e_enable_txqsq(&c->sq[tc]);
@@ -2272,7 +2275,7 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c)
mlx5e_activate_xsk(c);
}
-static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
+static void mlx5e_disable_channel(struct mlx5e_channel *c)
{
int tc;
@@ -2285,15 +2288,17 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
mlx5e_deactivate_xdpsq(&c->rq_xdpsq);
mlx5e_deactivate_icosq(&c->async_icosq);
mlx5e_deactivate_icosq(&c->icosq);
- synchronize_net(); /* Sync with NAPI. */
- for (tc = 0; tc < c->num_tc; tc++) {
+ for (tc = 0; tc < c->num_tc; tc++)
mlx5e_disable_txqsq(&c->sq[tc]);
- /* Sync with NAPI to prevent netif_tx_wake_queue. */
- synchronize_net();
- mlx5e_stop_txqsq(&c->sq[tc]);
- }
mlx5e_qos_deactivate_queues(c, false);
- synchronize_net();
+}
+
+static void mlx5e_stop_channel(struct mlx5e_channel *c)
+{
+ int tc;
+
+ for (tc = 0; tc < c->num_tc; tc++)
+ mlx5e_stop_txqsq(&c->sq[tc]);
mlx5e_qos_deactivate_queues(c, true);
napi_disable(&c->napi);
}
@@ -2371,11 +2376,15 @@ static void mlx5e_activate_channels(struct mlx5e_channels *chs)
{
int i;
- for (i = 0; i < chs->num; i++)
- mlx5e_activate_channel(chs->c[i]);
+ for (i = 0; i < chs->num; i++) {
+ mlx5e_enable_channel(chs->c[i]);
+ mlx5e_start_channel(chs->c[i]);
+ }
- if (chs->ptp)
- mlx5e_ptp_activate_channel(chs->ptp);
+ if (chs->ptp) {
+ mlx5e_ptp_enable_channel(chs->ptp);
+ mlx5e_ptp_start_channel(chs->ptp);
+ }
}
#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
@@ -2403,10 +2412,19 @@ static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
int i;
if (chs->ptp)
- mlx5e_ptp_deactivate_channel(chs->ptp);
+ mlx5e_ptp_disable_channel(chs->ptp);
+
+ for (i = 0; i < chs->num; i++)
+ mlx5e_disable_channel(chs->c[i]);
+
+ /* Sync with all NAPIs to wait until they stop using queues. */
+ synchronize_net();
+
+ if (chs->ptp)
+ mlx5e_ptp_stop_channel(chs->ptp);
for (i = 0; i < chs->num; i++)
- mlx5e_deactivate_channel(chs->c[i]);
+ mlx5e_stop_channel(chs->c[i]);
}
void mlx5e_close_channels(struct mlx5e_channels *chs)
--
2.31.1
next prev parent reply other threads:[~2021-11-19 19:58 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-19 19:58 [pull request][net 00/10] mlx5 fixes 2021-11-19 Saeed Mahameed
2021-11-19 19:58 ` [net 01/10] net/mlx5: E-switch, Respect BW share of the new group Saeed Mahameed
2021-11-19 19:58 ` [net 02/10] net/mlx5e: IPsec: Fix Software parser inner l3 type setting in case of encapsulation Saeed Mahameed
2021-11-19 19:58 ` [net 03/10] net/mlx5: Fix too early queueing of log timestamp work Saeed Mahameed
2021-11-19 19:58 ` [net 04/10] net/mlx5: Fix access to a non-supported register Saeed Mahameed
2021-11-19 19:58 ` [net 05/10] net/mlx5e: SHAMPO, Fix constant expression result Saeed Mahameed
2021-11-19 19:58 ` [net 06/10] net/mlx5: E-Switch, fix single FDB creation on BlueField Saeed Mahameed
2021-11-19 19:58 ` [net 07/10] net/mlx5e: Fix missing IPsec statistics on uplink representor Saeed Mahameed
2021-11-19 19:58 ` [net 08/10] net/mlx5e: Add activate/deactivate stage to XDPSQ Saeed Mahameed
2021-11-20 4:00 ` Jakub Kicinski
2021-11-19 19:58 ` [net 09/10] net/mlx5e: Call synchronize_net outside of deactivating a queue Saeed Mahameed
2021-11-19 19:58 ` Saeed Mahameed [this message]
2021-11-20 4:03 ` [net 10/10] net/mlx5e: Do synchronize_net only once when deactivating channels Jakub Kicinski
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211119195813.739586-11-saeed@kernel.org \
--to=saeed@kernel.org \
--cc=davem@davemloft.net \
--cc=kuba@kernel.org \
--cc=lkayal@nvidia.com \
--cc=maximmi@nvidia.com \
--cc=netdev@vger.kernel.org \
--cc=saeedm@nvidia.com \
--cc=tariqt@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).