From: Mark Bloch <mbloch@nvidia.com>
To: Eric Dumazet <edumazet@google.com>,
Jakub Kicinski <kuba@kernel.org>, Paolo Abeni <pabeni@redhat.com>,
Andrew Lunn <andrew+netdev@lunn.ch>,
"David S. Miller" <davem@davemloft.net>
Cc: Tariq Toukan <tariqt@nvidia.com>,
Leon Romanovsky <leon@kernel.org>,
"Saeed Mahameed" <saeedm@nvidia.com>, <netdev@vger.kernel.org>,
<linux-kernel@vger.kernel.org>, Gal Pressman <gal@nvidia.com>,
<linux-rdma@vger.kernel.org>, Moshe Shemesh <moshe@nvidia.com>,
Mark Bloch <mbloch@nvidia.com>, Shay Drory <shayd@nvidia.com>
Subject: [PATCH net V2 06/11] net/mlx5: Fix lockdep assertion on sync reset unload event
Date: Mon, 25 Aug 2025 17:34:29 +0300 [thread overview]
Message-ID: <20250825143435.598584-7-mbloch@nvidia.com> (raw)
In-Reply-To: <20250825143435.598584-1-mbloch@nvidia.com>
From: Moshe Shemesh <moshe@nvidia.com>
Fix lockdep assertion triggered during sync reset unload event. When the
sync reset flow is initiated using the devlink reload fw_activate
option, the PF already holds the devlink lock while handling unload
event. In this case, delegate sync reset unload event handling back to
the devlink callback process to avoid double-locking and resolve the
lockdep warning.
Kernel log:
WARNING: CPU: 9 PID: 1578 at devl_assert_locked+0x31/0x40
[...]
Call Trace:
<TASK>
mlx5_unload_one_devl_locked+0x2c/0xc0 [mlx5_core]
mlx5_sync_reset_unload_event+0xaf/0x2f0 [mlx5_core]
process_one_work+0x222/0x640
worker_thread+0x199/0x350
kthread+0x10b/0x230
? __pfx_worker_thread+0x10/0x10
? __pfx_kthread+0x10/0x10
ret_from_fork+0x8e/0x100
? __pfx_kthread+0x10/0x10
ret_from_fork_asm+0x1a/0x30
</TASK>
Fixes: 7a9770f1bfea ("net/mlx5: Handle sync reset unload event")
Signed-off-by: Moshe Shemesh <moshe@nvidia.com>
Signed-off-by: Mark Bloch <mbloch@nvidia.com>
---
.../net/ethernet/mellanox/mlx5/core/devlink.c | 2 +-
.../ethernet/mellanox/mlx5/core/fw_reset.c | 120 ++++++++++--------
.../ethernet/mellanox/mlx5/core/fw_reset.h | 1 +
3 files changed, 69 insertions(+), 54 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index 26091e7536d3..2c0e0c16ca90 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -160,7 +160,7 @@ static int mlx5_devlink_reload_fw_activate(struct devlink *devlink, struct netli
if (err)
return err;
- mlx5_unload_one_devl_locked(dev, false);
+ mlx5_sync_reset_unload_flow(dev, true);
err = mlx5_health_wait_pci_up(dev);
if (err)
NL_SET_ERR_MSG_MOD(extack, "FW activate aborted, PCI reads fail after reset");
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
index 69933addd921..38b9b184ae01 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
@@ -12,7 +12,8 @@ enum {
MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
MLX5_FW_RESET_FLAGS_PENDING_COMP,
MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS,
- MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED
+ MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED,
+ MLX5_FW_RESET_FLAGS_UNLOAD_EVENT,
};
struct mlx5_fw_reset {
@@ -219,7 +220,7 @@ int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev)
return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false);
}
-static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev, bool unloaded)
+static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev)
{
struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
struct devlink *devlink = priv_to_devlink(dev);
@@ -228,8 +229,7 @@ static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev, bool unload
if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) {
complete(&fw_reset->done);
} else {
- if (!unloaded)
- mlx5_unload_one(dev, false);
+ mlx5_sync_reset_unload_flow(dev, false);
if (mlx5_health_wait_pci_up(dev))
mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
else
@@ -272,7 +272,7 @@ static void mlx5_sync_reset_reload_work(struct work_struct *work)
mlx5_sync_reset_clear_reset_requested(dev, false);
mlx5_enter_error_state(dev, true);
- mlx5_fw_reset_complete_reload(dev, false);
+ mlx5_fw_reset_complete_reload(dev);
}
#define MLX5_RESET_POLL_INTERVAL (HZ / 10)
@@ -586,6 +586,65 @@ static int mlx5_sync_pci_reset(struct mlx5_core_dev *dev, u8 reset_method)
return err;
}
+void mlx5_sync_reset_unload_flow(struct mlx5_core_dev *dev, bool locked)
+{
+ struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
+ unsigned long timeout;
+ int poll_freq = 20;
+ bool reset_action;
+ u8 rst_state;
+ int err;
+
+ if (locked)
+ mlx5_unload_one_devl_locked(dev, false);
+ else
+ mlx5_unload_one(dev, false);
+
+ if (!test_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags))
+ return;
+
+ mlx5_set_fw_rst_ack(dev);
+ mlx5_core_warn(dev, "Sync Reset Unload done, device reset expected\n");
+
+ reset_action = false;
+ timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, RESET_UNLOAD));
+ do {
+ rst_state = mlx5_get_fw_rst_state(dev);
+ if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ ||
+ rst_state == MLX5_FW_RST_STATE_IDLE) {
+ reset_action = true;
+ break;
+ }
+ if (rst_state == MLX5_FW_RST_STATE_DROP_MODE) {
+ mlx5_core_info(dev, "Sync Reset Drop mode ack\n");
+ mlx5_set_fw_rst_ack(dev);
+ poll_freq = 1000;
+ }
+ msleep(poll_freq);
+ } while (!time_after(jiffies, timeout));
+
+ if (!reset_action) {
+ mlx5_core_err(dev, "Got timeout waiting for sync reset action, state = %u\n",
+ rst_state);
+ fw_reset->ret = -ETIMEDOUT;
+ goto done;
+ }
+
+ mlx5_core_warn(dev, "Sync Reset, got reset action. rst_state = %u\n",
+ rst_state);
+ if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ) {
+ err = mlx5_sync_pci_reset(dev, fw_reset->reset_method);
+ if (err) {
+ mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, err %d\n",
+ err);
+ fw_reset->ret = err;
+ }
+ }
+
+done:
+ clear_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags);
+}
+
static void mlx5_sync_reset_now_event(struct work_struct *work)
{
struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
@@ -613,17 +672,13 @@ static void mlx5_sync_reset_now_event(struct work_struct *work)
mlx5_enter_error_state(dev, true);
done:
fw_reset->ret = err;
- mlx5_fw_reset_complete_reload(dev, false);
+ mlx5_fw_reset_complete_reload(dev);
}
static void mlx5_sync_reset_unload_event(struct work_struct *work)
{
struct mlx5_fw_reset *fw_reset;
struct mlx5_core_dev *dev;
- unsigned long timeout;
- int poll_freq = 20;
- bool reset_action;
- u8 rst_state;
int err;
fw_reset = container_of(work, struct mlx5_fw_reset, reset_unload_work);
@@ -632,6 +687,7 @@ static void mlx5_sync_reset_unload_event(struct work_struct *work)
if (mlx5_sync_reset_clear_reset_requested(dev, false))
return;
+ set_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags);
mlx5_core_warn(dev, "Sync Reset Unload. Function is forced down.\n");
err = mlx5_cmd_fast_teardown_hca(dev);
@@ -640,49 +696,7 @@ static void mlx5_sync_reset_unload_event(struct work_struct *work)
else
mlx5_enter_error_state(dev, true);
- if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags))
- mlx5_unload_one_devl_locked(dev, false);
- else
- mlx5_unload_one(dev, false);
-
- mlx5_set_fw_rst_ack(dev);
- mlx5_core_warn(dev, "Sync Reset Unload done, device reset expected\n");
-
- reset_action = false;
- timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, RESET_UNLOAD));
- do {
- rst_state = mlx5_get_fw_rst_state(dev);
- if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ ||
- rst_state == MLX5_FW_RST_STATE_IDLE) {
- reset_action = true;
- break;
- }
- if (rst_state == MLX5_FW_RST_STATE_DROP_MODE) {
- mlx5_core_info(dev, "Sync Reset Drop mode ack\n");
- mlx5_set_fw_rst_ack(dev);
- poll_freq = 1000;
- }
- msleep(poll_freq);
- } while (!time_after(jiffies, timeout));
-
- if (!reset_action) {
- mlx5_core_err(dev, "Got timeout waiting for sync reset action, state = %u\n",
- rst_state);
- fw_reset->ret = -ETIMEDOUT;
- goto done;
- }
-
- mlx5_core_warn(dev, "Sync Reset, got reset action. rst_state = %u\n", rst_state);
- if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ) {
- err = mlx5_sync_pci_reset(dev, fw_reset->reset_method);
- if (err) {
- mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, err %d\n", err);
- fw_reset->ret = err;
- }
- }
-
-done:
- mlx5_fw_reset_complete_reload(dev, true);
+ mlx5_fw_reset_complete_reload(dev);
}
static void mlx5_sync_reset_abort_event(struct work_struct *work)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h
index ea527d06a85f..d5b28525c960 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h
@@ -12,6 +12,7 @@ int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel,
int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev);
int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev);
+void mlx5_sync_reset_unload_flow(struct mlx5_core_dev *dev, bool locked);
int mlx5_fw_reset_verify_fw_complete(struct mlx5_core_dev *dev,
struct netlink_ext_ack *extack);
void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev);
--
2.34.1
next prev parent reply other threads:[~2025-08-25 14:35 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-08-25 14:34 [PATCH net V2 00/11] mlx5 misc fixes 2025-08-25 Mark Bloch
2025-08-25 14:34 ` [PATCH net V2 01/11] net/mlx5: HWS, Fix memory leak in hws_pool_buddy_init error path Mark Bloch
2025-08-25 14:34 ` [PATCH net V2 02/11] net/mlx5: HWS, Fix memory leak in hws_action_get_shared_stc_nic error flow Mark Bloch
2025-08-25 14:34 ` [PATCH net V2 03/11] net/mlx5: HWS, Fix uninitialized variables in mlx5hws_pat_calc_nop " Mark Bloch
2025-08-25 14:34 ` [PATCH net V2 04/11] net/mlx5: HWS, Fix pattern destruction in mlx5hws_pat_get_pattern error path Mark Bloch
2025-08-25 14:34 ` [PATCH net V2 05/11] net/mlx5: Reload auxiliary drivers on fw_activate Mark Bloch
2025-08-25 14:34 ` Mark Bloch [this message]
2025-08-25 14:34 ` [PATCH net V2 07/11] net/mlx5: Nack sync reset when SFs are present Mark Bloch
2025-08-25 14:34 ` [PATCH net V2 08/11] net/mlx5: Prevent flow steering mode changes in switchdev mode Mark Bloch
2025-08-25 14:34 ` [PATCH net V2 09/11] net/mlx5e: Update and set Xon/Xoff upon MTU set Mark Bloch
2025-08-25 14:34 ` [PATCH net V2 10/11] net/mlx5e: Update and set Xon/Xoff upon port speed set Mark Bloch
2025-09-11 0:00 ` Jakub Kicinski
2025-09-11 13:47 ` Jakub Kicinski
2025-09-11 14:25 ` Mark Bloch
2025-09-11 14:36 ` Jakub Kicinski
2025-09-15 7:38 ` Tariq Toukan
2025-09-17 10:39 ` Tariq Toukan
2025-09-17 13:00 ` Daniel Zahka
2025-09-17 13:38 ` Tariq Toukan
2025-08-25 14:34 ` [PATCH net V2 11/11] net/mlx5e: Set local Xoff after FW update Mark Bloch
2025-08-27 1:10 ` [PATCH net V2 00/11] mlx5 misc fixes 2025-08-25 patchwork-bot+netdevbpf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20250825143435.598584-7-mbloch@nvidia.com \
--to=mbloch@nvidia.com \
--cc=andrew+netdev@lunn.ch \
--cc=davem@davemloft.net \
--cc=edumazet@google.com \
--cc=gal@nvidia.com \
--cc=kuba@kernel.org \
--cc=leon@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=moshe@nvidia.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=saeedm@nvidia.com \
--cc=shayd@nvidia.com \
--cc=tariqt@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox