From: Tariq Toukan <tariqt@nvidia.com>
To: Andrew Lunn <andrew+netdev@lunn.ch>,
"David S. Miller" <davem@davemloft.net>,
Eric Dumazet <edumazet@kernel.org>,
Jakub Kicinski <kuba@kernel.org>, <netdev@vger.kernel.org>,
Paolo Abeni <pabeni@redhat.com>
Cc: Carolina Jubran <cjubran@nvidia.com>,
Cosmin Ratiu <cratiu@nvidia.com>, Gal Pressman <gal@nvidia.com>,
Leon Romanovsky <leon@kernel.org>,
open list <linux-kernel@vger.kernel.org>,
<linux-rdma@vger.kernel.org>, Mark Bloch <mbloch@nvidia.com>,
Moshe Shemesh <moshe@nvidia.com>,
Or Har-Toov <ohartoov@nvidia.com>,
Patrisious Haddad <phaddad@nvidia.com>,
Saeed Mahameed <saeedm@nvidia.com>, Shay Drory <shayd@nvidia.com>,
Tariq Toukan <tariqt@nvidia.com>, Uri Dar <udar@nvidia.com>
Subject: [PATCH net-next V2 02/10] net/mlx5: LAG, select the MPESW master from the live member snapshot
Date: Mon, 16 Nov 2026 12:30:02 +0200 [thread overview]
Message-ID: <20261116103010.1129666-3-tariqt@nvidia.com> (raw)
In-Reply-To: <20261116103010.1129666-1-tariqt@nvidia.com>
From: Shay Drory <shayd@nvidia.com>
The LAG master is marked by the devcom PAIR event when all physical
ports have registered (mlx5_lag_num_devs() == ldev->ports). That does
not work for a virtual LAG of VFs/SFs: those functions are unrelated to
the physical port count.
Select the master from the current membership snapshot in
mlx5_lag_enable_mpesw() via mlx5_virt_lag_mark_master(), and clear it in
mlx5_lag_disable_mpesw(). Unlike the regular path, it walks all LAG
members.
mlx5_get_dev_index() is identical for a PF and all its VFs/SFs, so
tie-break by a stable per-member key is used - PCI BDF for VFs, sfnum
for SFs.
This is gated by a new ldev->virt_lag bit identifying this type of LAG.
Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Moshe Shemesh <moshe@nvidia.com>
Reviewed-by: Mark Bloch <mbloch@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
.../net/ethernet/mellanox/mlx5/core/lag/lag.c | 46 +++++++++++++++++--
.../net/ethernet/mellanox/mlx5/core/lag/lag.h | 3 ++
.../ethernet/mellanox/mlx5/core/lag/mpesw.c | 42 +++++++++++++----
.../ethernet/mellanox/mlx5/core/sf/dev/dev.h | 12 +++++
4 files changed, 90 insertions(+), 13 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index 9330aab24c02..c7ece82fd457 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -39,6 +39,7 @@
#include "lib/mlx5.h"
#include "lib/devcom.h"
#include "mlx5_core.h"
+#include "sf/dev/dev.h"
#include "eswitch.h"
#include "esw/acl/ofld.h"
#include "lag.h"
@@ -598,7 +599,43 @@ static void mlx5_lag_mark_master(struct mlx5_lag *ldev)
xa_set_mark(&ldev->pfs, master_xa_idx, MLX5_LAG_XA_MARK_MASTER);
}
-static void mlx5_lag_clear_master(struct mlx5_lag *ldev)
+static u64 mlx5_lag_member_tiebreak(struct mlx5_core_dev *dev)
+{
+ struct pci_dev *pdev = dev->pdev;
+
+ if (mlx5_core_is_sf(dev))
+ return mlx5_sf_coredev_sfnum(dev);
+
+ return ((u64)pci_domain_nr(pdev->bus) << 16) | pci_dev_id(pdev);
+}
+
+void mlx5_virt_lag_mark_master(struct mlx5_lag *ldev)
+{
+ u64 lowest_tiebreak = U64_MAX;
+ int lowest_dev_idx = INT_MAX;
+ int master_xa_idx = -1;
+ struct lag_func *pf;
+ u64 tiebreak;
+ int dev_idx;
+ int i;
+
+ mlx5_lag_for_each(i, 0, ldev, MLX5_LAG_FILTER_ALL) {
+ pf = mlx5_lag_pf(ldev, i);
+ dev_idx = mlx5_get_dev_index(pf->dev);
+ tiebreak = mlx5_lag_member_tiebreak(pf->dev);
+ if (dev_idx < lowest_dev_idx ||
+ (dev_idx == lowest_dev_idx && tiebreak < lowest_tiebreak)) {
+ lowest_dev_idx = dev_idx;
+ lowest_tiebreak = tiebreak;
+ master_xa_idx = i;
+ }
+ }
+
+ if (master_xa_idx >= 0)
+ xa_set_mark(&ldev->pfs, master_xa_idx, MLX5_LAG_XA_MARK_MASTER);
+}
+
+void mlx5_lag_clear_master(struct mlx5_lag *ldev)
{
unsigned long idx = 0;
void *entry;
@@ -618,7 +655,7 @@ static int mlx5_lag_devcom_event(int event, void *my_data, void *event_data)
int idx;
ldev = mlx5_lag_dev(dev);
- if (!ldev)
+ if (!ldev || ldev->virt_lag)
return 0;
mutex_lock(&ldev->lock);
@@ -1784,10 +1821,13 @@ struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev)
{
struct mlx5_devcom_comp_dev *devcom = NULL;
struct lag_func *pf;
+ u32 filter;
int i;
mutex_lock(&ldev->lock);
- i = mlx5_get_next_lag_func(ldev, 0, MLX5_LAG_FILTER_PORTS);
+ filter = ldev->virt_lag ? MLX5_LAG_FILTER_ALL :
+ MLX5_LAG_FILTER_PORTS;
+ i = mlx5_get_next_lag_func(ldev, 0, filter);
if (i < MLX5_MAX_PORTS) {
pf = mlx5_lag_pf(ldev, i);
devcom = pf->dev->priv.hca_devcom_comp;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
index 8552792b9035..21c83f51fcb5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
@@ -84,6 +84,7 @@ struct mlx5_lag {
unsigned long state_flags;
u8 ports;
u8 buckets;
+ bool virt_lag;
int mode_changes_in_progress;
u8 *v2p_map;
struct kref ref;
@@ -225,6 +226,8 @@ static inline bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev)
}
#endif
bool mlx5_lag_check_prereq(struct mlx5_lag *ldev);
+void mlx5_virt_lag_mark_master(struct mlx5_lag *ldev);
+void mlx5_lag_clear_master(struct mlx5_lag *ldev);
bool mlx5_lag_is_sd(struct mlx5_core_dev *dev);
int mlx5_lag_demux_init(struct mlx5_core_dev *dev,
struct mlx5_flow_table_attr *ft_attr);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
index 549d31812c2f..0bf5b4754a44 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
@@ -109,8 +109,8 @@ static bool mlx5_lag_has_sd_group(struct mlx5_lag *ldev)
static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
{
- int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
struct mlx5_core_dev *dev0;
+ int idx;
int err;
if (ldev->mode == MLX5_LAG_MODE_MPESW)
@@ -119,20 +119,32 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
if (ldev->mode != MLX5_LAG_MODE_NONE)
return -EINVAL;
- if (idx < 0)
- return -EINVAL;
+ /* VFs/SFs (nested LAG) are not marked by the LAG devcom PAIR event;
+ * select the master from the current snapshot before resolving it via
+ * mlx5_lag_get_dev_index_by_seq().
+ */
+ if (ldev->virt_lag)
+ mlx5_virt_lag_mark_master(ldev);
+
+ idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
+ if (idx < 0) {
+ err = -EINVAL;
+ goto err_clear_master;
+ }
dev0 = mlx5_lag_pf(ldev, idx)->dev;
if (mlx5_eswitch_mode(dev0) != MLX5_ESWITCH_OFFLOADS ||
!MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table) ||
!MLX5_CAP_GEN(dev0, create_lag_when_not_master_up) ||
!mlx5_lag_check_prereq(ldev) ||
- !mlx5_lag_shared_fdb_supported_filter(ldev, MLX5_LAG_FILTER_ALL))
- return -EOPNOTSUPP;
+ !mlx5_lag_shared_fdb_supported_filter(ldev, MLX5_LAG_FILTER_ALL)) {
+ err = -EOPNOTSUPP;
+ goto err_clear_master;
+ }
err = mlx5_mpesw_metadata_set(ldev);
if (err)
- return err;
+ goto err_clear_master;
if (mlx5_lag_has_sd_group(ldev))
mlx5_mpesw_teardown_sd_fdb(ldev);
@@ -143,13 +155,19 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
mlx5_core_warn(dev0,
"Failed to create LAG in MPESW mode (%d)\n",
err);
- if (mlx5_lag_has_sd_group(ldev))
- mlx5_mpesw_restore_sd_fdb(ldev);
- mlx5_mpesw_metadata_cleanup(ldev);
- return err;
+ goto err_restore;
}
return 0;
+
+err_restore:
+ if (mlx5_lag_has_sd_group(ldev))
+ mlx5_mpesw_restore_sd_fdb(ldev);
+ mlx5_mpesw_metadata_cleanup(ldev);
+err_clear_master:
+ if (ldev->virt_lag)
+ mlx5_lag_clear_master(ldev);
+ return err;
}
void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev)
@@ -161,6 +179,10 @@ void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev)
mlx5_lag_shared_fdb_destroy(ldev, MLX5_LAG_FILTER_ALL);
if (mlx5_lag_has_sd_group(ldev))
mlx5_mpesw_restore_sd_fdb(ldev);
+
+ /* Pair with the master mark taken at MPESW create for VFs. */
+ if (ldev->virt_lag)
+ mlx5_lag_clear_master(ldev);
}
void mlx5_mpesw_sd_devcoms_lock(struct mlx5_lag *ldev)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h
index 3ab0449c770c..382162baabdb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h
@@ -19,6 +19,13 @@ struct mlx5_sf_dev {
u16 fn_id;
};
+static inline u32 mlx5_sf_coredev_sfnum(struct mlx5_core_dev *mdev)
+{
+ struct auxiliary_device *adev = mlx5_sf_coredev_to_adev(mdev);
+
+ return container_of(adev, struct mlx5_sf_dev, adev)->sfnum;
+}
+
struct mlx5_sf_peer_devlink_event_ctx {
u16 fn_id;
struct devlink *devlink;
@@ -68,6 +75,11 @@ static inline bool mlx5_sf_dev_allocated(const struct mlx5_core_dev *dev)
return false;
}
+static inline u32 mlx5_sf_coredev_sfnum(struct mlx5_core_dev *mdev)
+{
+ return 0;
+}
+
#endif
#endif
--
2.44.0
next prev parent reply other threads:[~2026-11-16 10:36 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-16 10:30 [PATCH net-next V2 00/10] net/mlx5: LAG, preparations for LAG over VFs/SFs Tariq Toukan
2026-11-16 10:30 ` [PATCH net-next V2 01/10] net/mlx5: Distinguish VFs/SFs in the SW system image GUID for LAG grouping Tariq Toukan
2026-11-17 10:36 ` sashiko-bot
2026-11-16 10:30 ` Tariq Toukan [this message]
2026-11-17 10:36 ` [PATCH net-next V2 02/10] net/mlx5: LAG, select the MPESW master from the live member snapshot sashiko-bot
2026-11-16 10:30 ` [PATCH net-next V2 03/10] net/mlx5: LAG, skip phys LAG prerequisites for a virt LAG Tariq Toukan
2026-11-17 10:36 ` sashiko-bot
2026-11-16 10:30 ` [PATCH net-next V2 04/10] net/mlx5: LAG, treat virt LAG as a software managed LAG Tariq Toukan
2026-11-17 10:36 ` sashiko-bot
2026-11-16 10:30 ` [PATCH net-next V2 05/10] net/mlx5: LAG, decouple iteration end-of-walk from MLX5_MAX_PORTS Tariq Toukan
2026-11-17 10:36 ` sashiko-bot
2026-11-16 10:30 ` [PATCH net-next V2 06/10] net/mlx5: LAG, rename PF naming to fn Tariq Toukan
2026-11-17 10:36 ` sashiko-bot
2026-11-16 10:30 ` [PATCH net-next V2 07/10] net/mlx5: LAG, resolve affinity indices only in queue-affinity mode Tariq Toukan
2026-11-17 10:36 ` sashiko-bot
2026-11-16 10:30 ` [PATCH net-next V2 08/10] net/mlx5: LAG: replace fixed MLX5_MAX_PORTS arrays with dynamic allocation Tariq Toukan
2026-11-17 10:36 ` sashiko-bot
2026-11-16 10:30 ` [PATCH net-next V2 09/10] net/mlx5: LAG, drop MLX5_MAX_PORTS scratch array from device rescan Tariq Toukan
2026-11-17 10:36 ` sashiko-bot
2026-11-16 10:30 ` [PATCH net-next V2 10/10] net/mlx5: LAG, do not add a device to an already-active virt LAG Tariq Toukan
2026-11-17 10:36 ` sashiko-bot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261116103010.1129666-3-tariqt@nvidia.com \
--to=tariqt@nvidia.com \
--cc=andrew+netdev@lunn.ch \
--cc=cjubran@nvidia.com \
--cc=cratiu@nvidia.com \
--cc=davem@davemloft.net \
--cc=edumazet@kernel.org \
--cc=gal@nvidia.com \
--cc=kuba@kernel.org \
--cc=leon@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=mbloch@nvidia.com \
--cc=moshe@nvidia.com \
--cc=netdev@vger.kernel.org \
--cc=ohartoov@nvidia.com \
--cc=pabeni@redhat.com \
--cc=phaddad@nvidia.com \
--cc=saeedm@nvidia.com \
--cc=shayd@nvidia.com \
--cc=udar@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.