[PATCH net 4/5] net/mlx5: Lag, limit aggregated TX speed by PCIe bandwidth

[Tariq Toukan <tariqt@nvidia.com>]

From: Or Har-Toov <ohartoov@nvidia.com>

FW initializes max_tx_speed to the minimum of the port speed and the
PCI bandwidth. Modifying it with a value computed by different rules
changes the meaning of the field rather than updating it.

Limit each uplink's contribution to the aggregated LAG TX speed by the
NIC's PCIe bandwidth and reset max_tx_speed to the same value when lag
is torn down.

Fixes: 50f1d188c580 ("net/mlx5: Propagate LAG effective max_tx_speed to vports")
Signed-off-by: Or Har-Toov <ohartoov@nvidia.com>
Reviewed-by: Shay Drori <shayd@nvidia.com>
Reviewed-by: Mark Bloch <mbloch@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
---
 drivers/net/ethernet/mellanox/mlx5/core/dev.c | 20 +++++++++++++++++++
 .../net/ethernet/mellanox/mlx5/core/lag/lag.c | 12 +++++++++++
 .../ethernet/mellanox/mlx5/core/mlx5_core.h   |  1 +
 3 files changed, 33 insertions(+)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
index df2e3ad01819..81f3dc3b8034 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
@@ -30,6 +30,7 @@
  * SOFTWARE.
  */
 
+#include <linux/bitfield.h>
 #include <linux/mlx5/driver.h>
 #include <linux/mlx5/eswitch.h>
 #include <linux/mlx5/mlx5_ifc_vdpa.h>
@@ -40,6 +41,25 @@
 
 static DEFINE_IDA(mlx5_adev_ida);
 
+#define MLX5_PCIE_MIN_SPEED_MBPS	(2500)
+
+u32 mlx5_pcie_bandwidth(struct mlx5_core_dev *dev)
+{
+	u16 lnksta;
+	int speed;
+	u32 width;
+
+	if (pcie_capability_read_word(dev->pdev, PCI_EXP_LNKSTA, &lnksta))
+		return 0;
+
+	width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
+	speed = pcie_link_speed_mbps(dev->pdev);
+	if (speed < MLX5_PCIE_MIN_SPEED_MBPS)
+		return 0;
+
+	return speed * width;
+}
+
 static bool is_eth_rep_supported(struct mlx5_core_dev *dev)
 {
 	if (!IS_ENABLED(CONFIG_MLX5_ESWITCH))
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index 76b574fa0d7a..fd91becd6848 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -1418,6 +1418,7 @@ static int mlx5_lag_get_devices_oper_speed(struct mlx5_lag *ldev,
 {
 	struct mlx5_core_dev *pf_mdev;
 	struct lag_func *pf;
+	u32 pci_bw;
 	int pf_idx;
 	bool mpesw;
 	u32 speed;
@@ -1452,6 +1453,9 @@ static int mlx5_lag_get_devices_oper_speed(struct mlx5_lag *ldev,
 			return ret;
 		}
 
+		pci_bw = mlx5_pcie_bandwidth(pf_mdev);
+		if (pci_bw)
+			speed = min(speed, pci_bw);
 		*sum_speed += speed;
 	}
 
@@ -1463,6 +1467,7 @@ static int mlx5_lag_get_devices_max_speed(struct mlx5_lag *ldev, u32 *max_speed)
 	struct mlx5_core_dev *pf_mdev;
 	struct lag_func *pf;
 	bool take_max;
+	u32 pci_bw;
 	int pf_idx;
 	u32 speed;
 	int ret;
@@ -1488,6 +1493,9 @@ static int mlx5_lag_get_devices_max_speed(struct mlx5_lag *ldev, u32 *max_speed)
 			return ret;
 		}
 
+		pci_bw = mlx5_pcie_bandwidth(pf_mdev);
+		if (pci_bw)
+			speed = min(speed, pci_bw);
 		*max_speed = take_max ?
 			max(*max_speed, speed) : *max_speed + speed;
 	}
@@ -1588,6 +1596,7 @@ void mlx5_lag_reset_vports_speed(struct mlx5_lag *ldev)
 {
 	struct mlx5_core_dev *mdev;
 	struct lag_func *pf;
+	u32 pci_bw;
 	u32 speed;
 	int pf_idx;
 	int ret;
@@ -1609,6 +1618,9 @@ void mlx5_lag_reset_vports_speed(struct mlx5_lag *ldev)
 			continue;
 		}
 
+		pci_bw = mlx5_pcie_bandwidth(mdev);
+		if (pci_bw)
+			speed = min(speed, pci_bw);
 		speed = speed / MLX5_MAX_TX_SPEED_UNIT;
 		mlx5_lag_modify_device_vports_speed(mdev, speed);
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index d6713a2ce676..555842085daf 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -281,6 +281,7 @@ void mlx5_events_cleanup(struct mlx5_core_dev *dev);
 void mlx5_events_start(struct mlx5_core_dev *dev);
 void mlx5_events_stop(struct mlx5_core_dev *dev);
 
+u32 mlx5_pcie_bandwidth(struct mlx5_core_dev *dev);
 int mlx5_adev_idx_alloc(void);
 void mlx5_adev_idx_free(int idx);
 void mlx5_adev_cleanup(struct mlx5_core_dev *dev);
-- 
2.44.0