* [iwl-next v1 1/7] ice: devlink PF MSI-X max and min parameter
2024-02-13 7:35 [iwl-next v1 0/7] ice: managing MSI-X in driver Michal Swiatkowski
@ 2024-02-13 7:35 ` Michal Swiatkowski
2024-02-13 9:03 ` Jiri Pirko
2024-02-13 7:35 ` [iwl-next v1 2/7] ice: remove splitting MSI-X between features Michal Swiatkowski
` (5 subsequent siblings)
6 siblings, 1 reply; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 7:35 UTC (permalink / raw)
To: intel-wired-lan
Cc: netdev, pawel.chmielewski, sridhar.samudrala, jacob.e.keller,
pio.raczynski, konrad.knitter, marcin.szycik, wojciech.drewek,
nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel, Michal Swiatkowski
Use generic devlink PF MSI-X parameter to allow user to change MSI-X
range.
Reviewed-by: Wojciech Drewek <wojciech.drewek@intel.com>
Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
---
drivers/net/ethernet/intel/ice/ice.h | 8 ++
drivers/net/ethernet/intel/ice/ice_devlink.c | 82 ++++++++++++++++++++
drivers/net/ethernet/intel/ice/ice_irq.c | 6 ++
3 files changed, 96 insertions(+)
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index c4127d5f2be3..24085f3c0966 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -94,6 +94,7 @@
#define ICE_MIN_LAN_TXRX_MSIX 1
#define ICE_MIN_LAN_OICR_MSIX 1
#define ICE_MIN_MSIX (ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_LAN_OICR_MSIX)
+#define ICE_MAX_MSIX 256
#define ICE_FDIR_MSIX 2
#define ICE_RDMA_NUM_AEQ_MSIX 4
#define ICE_MIN_RDMA_MSIX 2
@@ -535,6 +536,12 @@ struct ice_agg_node {
u8 valid;
};
+struct ice_pf_msix {
+ u16 cur;
+ u16 min;
+ u16 max;
+};
+
struct ice_pf {
struct pci_dev *pdev;
@@ -604,6 +611,7 @@ struct ice_pf {
struct msi_map ll_ts_irq; /* LL_TS interrupt MSIX vector */
u16 max_pf_txqs; /* Total Tx queues PF wide */
u16 max_pf_rxqs; /* Total Rx queues PF wide */
+ struct ice_pf_msix msix;
u16 num_lan_msix; /* Total MSIX vectors for base driver */
u16 num_lan_tx; /* num LAN Tx queues setup */
u16 num_lan_rx; /* num LAN Rx queues setup */
diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
index cc717175178b..b82ff9556a4b 100644
--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
@@ -1603,6 +1603,78 @@ enum ice_param_id {
ICE_DEVLINK_PARAM_ID_LOOPBACK,
};
+static int
+ice_devlink_msix_max_pf_get(struct devlink *devlink, u32 id,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct ice_pf *pf = devlink_priv(devlink);
+
+ ctx->val.vu16 = pf->msix.max;
+
+ return 0;
+}
+
+static int
+ice_devlink_msix_max_pf_set(struct devlink *devlink, u32 id,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct ice_pf *pf = devlink_priv(devlink);
+ u16 max = ctx->val.vu16;
+
+ pf->msix.max = max;
+
+ return 0;
+}
+
+static int
+ice_devlink_msix_max_pf_validate(struct devlink *devlink, u32 id,
+ union devlink_param_value val,
+ struct netlink_ext_ack *extack)
+{
+ if (val.vu16 > ICE_MAX_MSIX) {
+ NL_SET_ERR_MSG_MOD(extack, "PF max MSI-X is too high");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int
+ice_devlink_msix_min_pf_get(struct devlink *devlink, u32 id,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct ice_pf *pf = devlink_priv(devlink);
+
+ ctx->val.vu16 = pf->msix.min;
+
+ return 0;
+}
+
+static int
+ice_devlink_msix_min_pf_set(struct devlink *devlink, u32 id,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct ice_pf *pf = devlink_priv(devlink);
+ u16 min = ctx->val.vu16;
+
+ pf->msix.min = min;
+
+ return 0;
+}
+
+static int
+ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
+ union devlink_param_value val,
+ struct netlink_ext_ack *extack)
+{
+ if (val.vu16 <= ICE_MIN_MSIX) {
+ NL_SET_ERR_MSG_MOD(extack, "PF min MSI-X is too low");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static const struct devlink_param ice_devlink_params[] = {
DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
ice_devlink_enable_roce_get,
@@ -1618,6 +1690,16 @@ static const struct devlink_param ice_devlink_params[] = {
ice_devlink_loopback_get,
ice_devlink_loopback_set,
ice_devlink_loopback_validate),
+ DEVLINK_PARAM_GENERIC(MSIX_VEC_PER_PF_MAX,
+ BIT(DEVLINK_PARAM_CMODE_PERMANENT),
+ ice_devlink_msix_max_pf_get,
+ ice_devlink_msix_max_pf_set,
+ ice_devlink_msix_max_pf_validate),
+ DEVLINK_PARAM_GENERIC(MSIX_VEC_PER_PF_MIN,
+ BIT(DEVLINK_PARAM_CMODE_PERMANENT),
+ ice_devlink_msix_min_pf_get,
+ ice_devlink_msix_min_pf_set,
+ ice_devlink_msix_min_pf_validate),
};
static void ice_devlink_free(void *devlink_ptr)
diff --git a/drivers/net/ethernet/intel/ice/ice_irq.c b/drivers/net/ethernet/intel/ice/ice_irq.c
index ad82ff7d1995..fa7178a68b94 100644
--- a/drivers/net/ethernet/intel/ice/ice_irq.c
+++ b/drivers/net/ethernet/intel/ice/ice_irq.c
@@ -254,6 +254,12 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
int total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
int vectors, max_vectors;
+ /* load default PF MSI-X range */
+ if (!pf->msix.min)
+ pf->msix.min = ICE_MIN_MSIX;
+ if (!pf->msix.max)
+ pf->msix.max = total_vectors / 2;
+
vectors = ice_ena_msix_range(pf);
if (vectors < 0)
--
2.42.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [iwl-next v1 1/7] ice: devlink PF MSI-X max and min parameter
2024-02-13 7:35 ` [iwl-next v1 1/7] ice: devlink PF MSI-X max and min parameter Michal Swiatkowski
@ 2024-02-13 9:03 ` Jiri Pirko
2024-02-13 10:01 ` Michal Swiatkowski
0 siblings, 1 reply; 17+ messages in thread
From: Jiri Pirko @ 2024-02-13 9:03 UTC (permalink / raw)
To: Michal Swiatkowski
Cc: intel-wired-lan, netdev, pawel.chmielewski, sridhar.samudrala,
jacob.e.keller, pio.raczynski, konrad.knitter, marcin.szycik,
wojciech.drewek, nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel
Tue, Feb 13, 2024 at 08:35:03AM CET, michal.swiatkowski@linux.intel.com wrote:
>Use generic devlink PF MSI-X parameter to allow user to change MSI-X
>range.
>
>Reviewed-by: Wojciech Drewek <wojciech.drewek@intel.com>
>Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
>---
> drivers/net/ethernet/intel/ice/ice.h | 8 ++
> drivers/net/ethernet/intel/ice/ice_devlink.c | 82 ++++++++++++++++++++
> drivers/net/ethernet/intel/ice/ice_irq.c | 6 ++
> 3 files changed, 96 insertions(+)
>
>diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
>index c4127d5f2be3..24085f3c0966 100644
>--- a/drivers/net/ethernet/intel/ice/ice.h
>+++ b/drivers/net/ethernet/intel/ice/ice.h
>@@ -94,6 +94,7 @@
> #define ICE_MIN_LAN_TXRX_MSIX 1
> #define ICE_MIN_LAN_OICR_MSIX 1
> #define ICE_MIN_MSIX (ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_LAN_OICR_MSIX)
>+#define ICE_MAX_MSIX 256
> #define ICE_FDIR_MSIX 2
> #define ICE_RDMA_NUM_AEQ_MSIX 4
> #define ICE_MIN_RDMA_MSIX 2
>@@ -535,6 +536,12 @@ struct ice_agg_node {
> u8 valid;
> };
>
>+struct ice_pf_msix {
>+ u16 cur;
>+ u16 min;
>+ u16 max;
>+};
>+
> struct ice_pf {
> struct pci_dev *pdev;
>
>@@ -604,6 +611,7 @@ struct ice_pf {
> struct msi_map ll_ts_irq; /* LL_TS interrupt MSIX vector */
> u16 max_pf_txqs; /* Total Tx queues PF wide */
> u16 max_pf_rxqs; /* Total Rx queues PF wide */
>+ struct ice_pf_msix msix;
> u16 num_lan_msix; /* Total MSIX vectors for base driver */
> u16 num_lan_tx; /* num LAN Tx queues setup */
> u16 num_lan_rx; /* num LAN Rx queues setup */
>diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
>index cc717175178b..b82ff9556a4b 100644
>--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
>+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
>@@ -1603,6 +1603,78 @@ enum ice_param_id {
> ICE_DEVLINK_PARAM_ID_LOOPBACK,
> };
>
>+static int
>+ice_devlink_msix_max_pf_get(struct devlink *devlink, u32 id,
>+ struct devlink_param_gset_ctx *ctx)
>+{
>+ struct ice_pf *pf = devlink_priv(devlink);
>+
>+ ctx->val.vu16 = pf->msix.max;
>+
>+ return 0;
>+}
>+
>+static int
>+ice_devlink_msix_max_pf_set(struct devlink *devlink, u32 id,
>+ struct devlink_param_gset_ctx *ctx)
>+{
>+ struct ice_pf *pf = devlink_priv(devlink);
>+ u16 max = ctx->val.vu16;
>+
>+ pf->msix.max = max;
What's permanent about this exactly?
>+
>+ return 0;
>+}
>+
>+static int
>+ice_devlink_msix_max_pf_validate(struct devlink *devlink, u32 id,
>+ union devlink_param_value val,
>+ struct netlink_ext_ack *extack)
>+{
>+ if (val.vu16 > ICE_MAX_MSIX) {
>+ NL_SET_ERR_MSG_MOD(extack, "PF max MSI-X is too high");
>+ return -EINVAL;
>+ }
>+
>+ return 0;
>+}
>+
>+static int
>+ice_devlink_msix_min_pf_get(struct devlink *devlink, u32 id,
>+ struct devlink_param_gset_ctx *ctx)
>+{
>+ struct ice_pf *pf = devlink_priv(devlink);
>+
>+ ctx->val.vu16 = pf->msix.min;
>+
>+ return 0;
>+}
>+
>+static int
>+ice_devlink_msix_min_pf_set(struct devlink *devlink, u32 id,
>+ struct devlink_param_gset_ctx *ctx)
>+{
>+ struct ice_pf *pf = devlink_priv(devlink);
>+ u16 min = ctx->val.vu16;
>+
>+ pf->msix.min = min;
What's permanent about this exactly?
>+
>+ return 0;
>+}
>+
>+static int
>+ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
>+ union devlink_param_value val,
>+ struct netlink_ext_ack *extack)
>+{
>+ if (val.vu16 <= ICE_MIN_MSIX) {
>+ NL_SET_ERR_MSG_MOD(extack, "PF min MSI-X is too low");
>+ return -EINVAL;
>+ }
>+
>+ return 0;
>+}
>+
> static const struct devlink_param ice_devlink_params[] = {
> DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
> ice_devlink_enable_roce_get,
>@@ -1618,6 +1690,16 @@ static const struct devlink_param ice_devlink_params[] = {
> ice_devlink_loopback_get,
> ice_devlink_loopback_set,
> ice_devlink_loopback_validate),
>+ DEVLINK_PARAM_GENERIC(MSIX_VEC_PER_PF_MAX,
>+ BIT(DEVLINK_PARAM_CMODE_PERMANENT),
>+ ice_devlink_msix_max_pf_get,
>+ ice_devlink_msix_max_pf_set,
>+ ice_devlink_msix_max_pf_validate),
>+ DEVLINK_PARAM_GENERIC(MSIX_VEC_PER_PF_MIN,
>+ BIT(DEVLINK_PARAM_CMODE_PERMANENT),
....
pw-bot: cr
>+ ice_devlink_msix_min_pf_get,
>+ ice_devlink_msix_min_pf_set,
>+ ice_devlink_msix_min_pf_validate),
> };
>
> static void ice_devlink_free(void *devlink_ptr)
>diff --git a/drivers/net/ethernet/intel/ice/ice_irq.c b/drivers/net/ethernet/intel/ice/ice_irq.c
>index ad82ff7d1995..fa7178a68b94 100644
>--- a/drivers/net/ethernet/intel/ice/ice_irq.c
>+++ b/drivers/net/ethernet/intel/ice/ice_irq.c
>@@ -254,6 +254,12 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
> int total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
> int vectors, max_vectors;
>
>+ /* load default PF MSI-X range */
>+ if (!pf->msix.min)
>+ pf->msix.min = ICE_MIN_MSIX;
>+ if (!pf->msix.max)
>+ pf->msix.max = total_vectors / 2;
>+
> vectors = ice_ena_msix_range(pf);
>
> if (vectors < 0)
>--
>2.42.0
>
>
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [iwl-next v1 1/7] ice: devlink PF MSI-X max and min parameter
2024-02-13 9:03 ` Jiri Pirko
@ 2024-02-13 10:01 ` Michal Swiatkowski
2024-02-13 11:31 ` Jiri Pirko
0 siblings, 1 reply; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 10:01 UTC (permalink / raw)
To: Jiri Pirko
Cc: intel-wired-lan, netdev, pawel.chmielewski, sridhar.samudrala,
jacob.e.keller, pio.raczynski, konrad.knitter, marcin.szycik,
wojciech.drewek, nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel
On Tue, Feb 13, 2024 at 10:03:05AM +0100, Jiri Pirko wrote:
> Tue, Feb 13, 2024 at 08:35:03AM CET, michal.swiatkowski@linux.intel.com wrote:
> >Use generic devlink PF MSI-X parameter to allow user to change MSI-X
> >range.
> >
> >Reviewed-by: Wojciech Drewek <wojciech.drewek@intel.com>
> >Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
> >---
> > drivers/net/ethernet/intel/ice/ice.h | 8 ++
> > drivers/net/ethernet/intel/ice/ice_devlink.c | 82 ++++++++++++++++++++
> > drivers/net/ethernet/intel/ice/ice_irq.c | 6 ++
> > 3 files changed, 96 insertions(+)
> >
> >diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
> >index c4127d5f2be3..24085f3c0966 100644
> >--- a/drivers/net/ethernet/intel/ice/ice.h
> >+++ b/drivers/net/ethernet/intel/ice/ice.h
> >@@ -94,6 +94,7 @@
> > #define ICE_MIN_LAN_TXRX_MSIX 1
> > #define ICE_MIN_LAN_OICR_MSIX 1
> > #define ICE_MIN_MSIX (ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_LAN_OICR_MSIX)
> >+#define ICE_MAX_MSIX 256
> > #define ICE_FDIR_MSIX 2
> > #define ICE_RDMA_NUM_AEQ_MSIX 4
> > #define ICE_MIN_RDMA_MSIX 2
> >@@ -535,6 +536,12 @@ struct ice_agg_node {
> > u8 valid;
> > };
> >
> >+struct ice_pf_msix {
> >+ u16 cur;
> >+ u16 min;
> >+ u16 max;
> >+};
> >+
> > struct ice_pf {
> > struct pci_dev *pdev;
> >
> >@@ -604,6 +611,7 @@ struct ice_pf {
> > struct msi_map ll_ts_irq; /* LL_TS interrupt MSIX vector */
> > u16 max_pf_txqs; /* Total Tx queues PF wide */
> > u16 max_pf_rxqs; /* Total Rx queues PF wide */
> >+ struct ice_pf_msix msix;
> > u16 num_lan_msix; /* Total MSIX vectors for base driver */
> > u16 num_lan_tx; /* num LAN Tx queues setup */
> > u16 num_lan_rx; /* num LAN Rx queues setup */
> >diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
> >index cc717175178b..b82ff9556a4b 100644
> >--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
> >+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
> >@@ -1603,6 +1603,78 @@ enum ice_param_id {
> > ICE_DEVLINK_PARAM_ID_LOOPBACK,
> > };
> >
> >+static int
> >+ice_devlink_msix_max_pf_get(struct devlink *devlink, u32 id,
> >+ struct devlink_param_gset_ctx *ctx)
> >+{
> >+ struct ice_pf *pf = devlink_priv(devlink);
> >+
> >+ ctx->val.vu16 = pf->msix.max;
> >+
> >+ return 0;
> >+}
> >+
> >+static int
> >+ice_devlink_msix_max_pf_set(struct devlink *devlink, u32 id,
> >+ struct devlink_param_gset_ctx *ctx)
> >+{
> >+ struct ice_pf *pf = devlink_priv(devlink);
> >+ u16 max = ctx->val.vu16;
> >+
> >+ pf->msix.max = max;
>
> What's permanent about this exactly?
>
I want to store the value here after driver reinit. Isn't it enough to
use this parameter type? Which one should be used for this purpose?
>
> >+
> >+ return 0;
> >+}
> >+
> >+static int
> >+ice_devlink_msix_max_pf_validate(struct devlink *devlink, u32 id,
> >+ union devlink_param_value val,
> >+ struct netlink_ext_ack *extack)
> >+{
> >+ if (val.vu16 > ICE_MAX_MSIX) {
> >+ NL_SET_ERR_MSG_MOD(extack, "PF max MSI-X is too high");
> >+ return -EINVAL;
> >+ }
> >+
> >+ return 0;
> >+}
> >+
> >+static int
> >+ice_devlink_msix_min_pf_get(struct devlink *devlink, u32 id,
> >+ struct devlink_param_gset_ctx *ctx)
> >+{
> >+ struct ice_pf *pf = devlink_priv(devlink);
> >+
> >+ ctx->val.vu16 = pf->msix.min;
> >+
> >+ return 0;
> >+}
> >+
> >+static int
> >+ice_devlink_msix_min_pf_set(struct devlink *devlink, u32 id,
> >+ struct devlink_param_gset_ctx *ctx)
> >+{
> >+ struct ice_pf *pf = devlink_priv(devlink);
> >+ u16 min = ctx->val.vu16;
> >+
> >+ pf->msix.min = min;
>
> What's permanent about this exactly?
>
The same as with max.
>
> >+
> >+ return 0;
> >+}
> >+
> >+static int
> >+ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
> >+ union devlink_param_value val,
> >+ struct netlink_ext_ack *extack)
> >+{
> >+ if (val.vu16 <= ICE_MIN_MSIX) {
> >+ NL_SET_ERR_MSG_MOD(extack, "PF min MSI-X is too low");
> >+ return -EINVAL;
> >+ }
> >+
> >+ return 0;
> >+}
> >+
> > static const struct devlink_param ice_devlink_params[] = {
> > DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
> > ice_devlink_enable_roce_get,
> >@@ -1618,6 +1690,16 @@ static const struct devlink_param ice_devlink_params[] = {
> > ice_devlink_loopback_get,
> > ice_devlink_loopback_set,
> > ice_devlink_loopback_validate),
> >+ DEVLINK_PARAM_GENERIC(MSIX_VEC_PER_PF_MAX,
> >+ BIT(DEVLINK_PARAM_CMODE_PERMANENT),
> >+ ice_devlink_msix_max_pf_get,
> >+ ice_devlink_msix_max_pf_set,
> >+ ice_devlink_msix_max_pf_validate),
> >+ DEVLINK_PARAM_GENERIC(MSIX_VEC_PER_PF_MIN,
> >+ BIT(DEVLINK_PARAM_CMODE_PERMANENT),
>
> ....
>
>
> pw-bot: cr
>
>
> >+ ice_devlink_msix_min_pf_get,
> >+ ice_devlink_msix_min_pf_set,
> >+ ice_devlink_msix_min_pf_validate),
> > };
> >
> > static void ice_devlink_free(void *devlink_ptr)
> >diff --git a/drivers/net/ethernet/intel/ice/ice_irq.c b/drivers/net/ethernet/intel/ice/ice_irq.c
> >index ad82ff7d1995..fa7178a68b94 100644
> >--- a/drivers/net/ethernet/intel/ice/ice_irq.c
> >+++ b/drivers/net/ethernet/intel/ice/ice_irq.c
> >@@ -254,6 +254,12 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
> > int total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
> > int vectors, max_vectors;
> >
> >+ /* load default PF MSI-X range */
> >+ if (!pf->msix.min)
> >+ pf->msix.min = ICE_MIN_MSIX;
> >+ if (!pf->msix.max)
> >+ pf->msix.max = total_vectors / 2;
> >+
> > vectors = ice_ena_msix_range(pf);
> >
> > if (vectors < 0)
> >--
> >2.42.0
> >
> >
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [iwl-next v1 1/7] ice: devlink PF MSI-X max and min parameter
2024-02-13 10:01 ` Michal Swiatkowski
@ 2024-02-13 11:31 ` Jiri Pirko
2024-02-13 11:51 ` Michal Swiatkowski
0 siblings, 1 reply; 17+ messages in thread
From: Jiri Pirko @ 2024-02-13 11:31 UTC (permalink / raw)
To: Michal Swiatkowski
Cc: intel-wired-lan, netdev, pawel.chmielewski, sridhar.samudrala,
jacob.e.keller, pio.raczynski, konrad.knitter, marcin.szycik,
wojciech.drewek, nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel
Tue, Feb 13, 2024 at 11:01:08AM CET, michal.swiatkowski@linux.intel.com wrote:
>On Tue, Feb 13, 2024 at 10:03:05AM +0100, Jiri Pirko wrote:
>> Tue, Feb 13, 2024 at 08:35:03AM CET, michal.swiatkowski@linux.intel.com wrote:
>> >Use generic devlink PF MSI-X parameter to allow user to change MSI-X
>> >range.
>> >
>> >Reviewed-by: Wojciech Drewek <wojciech.drewek@intel.com>
>> >Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
>> >---
>> > drivers/net/ethernet/intel/ice/ice.h | 8 ++
>> > drivers/net/ethernet/intel/ice/ice_devlink.c | 82 ++++++++++++++++++++
>> > drivers/net/ethernet/intel/ice/ice_irq.c | 6 ++
>> > 3 files changed, 96 insertions(+)
>> >
>> >diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
>> >index c4127d5f2be3..24085f3c0966 100644
>> >--- a/drivers/net/ethernet/intel/ice/ice.h
>> >+++ b/drivers/net/ethernet/intel/ice/ice.h
>> >@@ -94,6 +94,7 @@
>> > #define ICE_MIN_LAN_TXRX_MSIX 1
>> > #define ICE_MIN_LAN_OICR_MSIX 1
>> > #define ICE_MIN_MSIX (ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_LAN_OICR_MSIX)
>> >+#define ICE_MAX_MSIX 256
>> > #define ICE_FDIR_MSIX 2
>> > #define ICE_RDMA_NUM_AEQ_MSIX 4
>> > #define ICE_MIN_RDMA_MSIX 2
>> >@@ -535,6 +536,12 @@ struct ice_agg_node {
>> > u8 valid;
>> > };
>> >
>> >+struct ice_pf_msix {
>> >+ u16 cur;
>> >+ u16 min;
>> >+ u16 max;
>> >+};
>> >+
>> > struct ice_pf {
>> > struct pci_dev *pdev;
>> >
>> >@@ -604,6 +611,7 @@ struct ice_pf {
>> > struct msi_map ll_ts_irq; /* LL_TS interrupt MSIX vector */
>> > u16 max_pf_txqs; /* Total Tx queues PF wide */
>> > u16 max_pf_rxqs; /* Total Rx queues PF wide */
>> >+ struct ice_pf_msix msix;
>> > u16 num_lan_msix; /* Total MSIX vectors for base driver */
>> > u16 num_lan_tx; /* num LAN Tx queues setup */
>> > u16 num_lan_rx; /* num LAN Rx queues setup */
>> >diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
>> >index cc717175178b..b82ff9556a4b 100644
>> >--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
>> >+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
>> >@@ -1603,6 +1603,78 @@ enum ice_param_id {
>> > ICE_DEVLINK_PARAM_ID_LOOPBACK,
>> > };
>> >
>> >+static int
>> >+ice_devlink_msix_max_pf_get(struct devlink *devlink, u32 id,
>> >+ struct devlink_param_gset_ctx *ctx)
>> >+{
>> >+ struct ice_pf *pf = devlink_priv(devlink);
>> >+
>> >+ ctx->val.vu16 = pf->msix.max;
>> >+
>> >+ return 0;
>> >+}
>> >+
>> >+static int
>> >+ice_devlink_msix_max_pf_set(struct devlink *devlink, u32 id,
>> >+ struct devlink_param_gset_ctx *ctx)
>> >+{
>> >+ struct ice_pf *pf = devlink_priv(devlink);
>> >+ u16 max = ctx->val.vu16;
>> >+
>> >+ pf->msix.max = max;
>>
>> What's permanent about this exactly?
>>
>
>I want to store the value here after driver reinit. Isn't it enough to
>use this parameter type? Which one should be used for this purpose?
Documentation/networking/devlink/devlink-params.rst say:
.. list-table:: Possible configuration modes
:widths: 5 90
* - Name
- Description
* - ``runtime``
- set while the driver is running, and takes effect immediately. No
reset is required.
* - ``driverinit``
- applied while the driver initializes. Requires the user to restart
the driver using the ``devlink`` reload command.
* - ``permanent``
- written to the device's non-volatile memory. A hard reset is required
for it to take effect.
[...]
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [iwl-next v1 1/7] ice: devlink PF MSI-X max and min parameter
2024-02-13 11:31 ` Jiri Pirko
@ 2024-02-13 11:51 ` Michal Swiatkowski
0 siblings, 0 replies; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 11:51 UTC (permalink / raw)
To: Jiri Pirko
Cc: intel-wired-lan, netdev, pawel.chmielewski, sridhar.samudrala,
jacob.e.keller, pio.raczynski, konrad.knitter, marcin.szycik,
wojciech.drewek, nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel
On Tue, Feb 13, 2024 at 12:31:59PM +0100, Jiri Pirko wrote:
> Tue, Feb 13, 2024 at 11:01:08AM CET, michal.swiatkowski@linux.intel.com wrote:
> >On Tue, Feb 13, 2024 at 10:03:05AM +0100, Jiri Pirko wrote:
> >> Tue, Feb 13, 2024 at 08:35:03AM CET, michal.swiatkowski@linux.intel.com wrote:
> >> >Use generic devlink PF MSI-X parameter to allow user to change MSI-X
> >> >range.
> >> >
> >> >Reviewed-by: Wojciech Drewek <wojciech.drewek@intel.com>
> >> >Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
> >> >---
> >> > drivers/net/ethernet/intel/ice/ice.h | 8 ++
> >> > drivers/net/ethernet/intel/ice/ice_devlink.c | 82 ++++++++++++++++++++
> >> > drivers/net/ethernet/intel/ice/ice_irq.c | 6 ++
> >> > 3 files changed, 96 insertions(+)
> >> >
> >> >diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
> >> >index c4127d5f2be3..24085f3c0966 100644
> >> >--- a/drivers/net/ethernet/intel/ice/ice.h
> >> >+++ b/drivers/net/ethernet/intel/ice/ice.h
> >> >@@ -94,6 +94,7 @@
> >> > #define ICE_MIN_LAN_TXRX_MSIX 1
> >> > #define ICE_MIN_LAN_OICR_MSIX 1
> >> > #define ICE_MIN_MSIX (ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_LAN_OICR_MSIX)
> >> >+#define ICE_MAX_MSIX 256
> >> > #define ICE_FDIR_MSIX 2
> >> > #define ICE_RDMA_NUM_AEQ_MSIX 4
> >> > #define ICE_MIN_RDMA_MSIX 2
> >> >@@ -535,6 +536,12 @@ struct ice_agg_node {
> >> > u8 valid;
> >> > };
> >> >
> >> >+struct ice_pf_msix {
> >> >+ u16 cur;
> >> >+ u16 min;
> >> >+ u16 max;
> >> >+};
> >> >+
> >> > struct ice_pf {
> >> > struct pci_dev *pdev;
> >> >
> >> >@@ -604,6 +611,7 @@ struct ice_pf {
> >> > struct msi_map ll_ts_irq; /* LL_TS interrupt MSIX vector */
> >> > u16 max_pf_txqs; /* Total Tx queues PF wide */
> >> > u16 max_pf_rxqs; /* Total Rx queues PF wide */
> >> >+ struct ice_pf_msix msix;
> >> > u16 num_lan_msix; /* Total MSIX vectors for base driver */
> >> > u16 num_lan_tx; /* num LAN Tx queues setup */
> >> > u16 num_lan_rx; /* num LAN Rx queues setup */
> >> >diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
> >> >index cc717175178b..b82ff9556a4b 100644
> >> >--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
> >> >+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
> >> >@@ -1603,6 +1603,78 @@ enum ice_param_id {
> >> > ICE_DEVLINK_PARAM_ID_LOOPBACK,
> >> > };
> >> >
> >> >+static int
> >> >+ice_devlink_msix_max_pf_get(struct devlink *devlink, u32 id,
> >> >+ struct devlink_param_gset_ctx *ctx)
> >> >+{
> >> >+ struct ice_pf *pf = devlink_priv(devlink);
> >> >+
> >> >+ ctx->val.vu16 = pf->msix.max;
> >> >+
> >> >+ return 0;
> >> >+}
> >> >+
> >> >+static int
> >> >+ice_devlink_msix_max_pf_set(struct devlink *devlink, u32 id,
> >> >+ struct devlink_param_gset_ctx *ctx)
> >> >+{
> >> >+ struct ice_pf *pf = devlink_priv(devlink);
> >> >+ u16 max = ctx->val.vu16;
> >> >+
> >> >+ pf->msix.max = max;
> >>
> >> What's permanent about this exactly?
> >>
> >
> >I want to store the value here after driver reinit. Isn't it enough to
> >use this parameter type? Which one should be used for this purpose?
>
> Documentation/networking/devlink/devlink-params.rst say:
>
> .. list-table:: Possible configuration modes
> :widths: 5 90
>
> * - Name
> - Description
> * - ``runtime``
> - set while the driver is running, and takes effect immediately. No
> reset is required.
> * - ``driverinit``
> - applied while the driver initializes. Requires the user to restart
> the driver using the ``devlink`` reload command.
> * - ``permanent``
> - written to the device's non-volatile memory. A hard reset is required
> for it to take effect.
>
>
> [...]
Thanks for pointing it, I changed the idea during developing it (at
first I wanted to store it in NVM) and forgot to change the type.
I will go with driverinit param.
^ permalink raw reply [flat|nested] 17+ messages in thread
* [iwl-next v1 2/7] ice: remove splitting MSI-X between features
2024-02-13 7:35 [iwl-next v1 0/7] ice: managing MSI-X in driver Michal Swiatkowski
2024-02-13 7:35 ` [iwl-next v1 1/7] ice: devlink PF MSI-X max and min parameter Michal Swiatkowski
@ 2024-02-13 7:35 ` Michal Swiatkowski
2024-02-13 7:35 ` [iwl-next v1 3/7] ice: get rid of num_lan_msix field Michal Swiatkowski
` (4 subsequent siblings)
6 siblings, 0 replies; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 7:35 UTC (permalink / raw)
To: intel-wired-lan
Cc: netdev, pawel.chmielewski, sridhar.samudrala, jacob.e.keller,
pio.raczynski, konrad.knitter, marcin.szycik, wojciech.drewek,
nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel, Michal Swiatkowski
With dynamic approach to alloc MSI-X there is no sense to statically
split MSI-X between PF features.
Reviewed-by: Wojciech Drewek <wojciech.drewek@intel.com>
Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
---
drivers/net/ethernet/intel/ice/ice_irq.c | 168 ++---------------------
1 file changed, 10 insertions(+), 158 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_irq.c b/drivers/net/ethernet/intel/ice/ice_irq.c
index fa7178a68b94..8116d1d2ffa0 100644
--- a/drivers/net/ethernet/intel/ice/ice_irq.c
+++ b/drivers/net/ethernet/intel/ice/ice_irq.c
@@ -84,157 +84,6 @@ static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only)
return entry;
}
-/**
- * ice_reduce_msix_usage - Reduce usage of MSI-X vectors
- * @pf: board private structure
- * @v_remain: number of remaining MSI-X vectors to be distributed
- *
- * Reduce the usage of MSI-X vectors when entire request cannot be fulfilled.
- * pf->num_lan_msix and pf->num_rdma_msix values are set based on number of
- * remaining vectors.
- */
-static void ice_reduce_msix_usage(struct ice_pf *pf, int v_remain)
-{
- int v_rdma;
-
- if (!ice_is_rdma_ena(pf)) {
- pf->num_lan_msix = v_remain;
- return;
- }
-
- /* RDMA needs at least 1 interrupt in addition to AEQ MSIX */
- v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1;
-
- if (v_remain < ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_RDMA_MSIX) {
- dev_warn(ice_pf_to_dev(pf), "Not enough MSI-X vectors to support RDMA.\n");
- clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
-
- pf->num_rdma_msix = 0;
- pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX;
- } else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) ||
- (v_remain - v_rdma < v_rdma)) {
- /* Support minimum RDMA and give remaining vectors to LAN MSIX
- */
- pf->num_rdma_msix = ICE_MIN_RDMA_MSIX;
- pf->num_lan_msix = v_remain - ICE_MIN_RDMA_MSIX;
- } else {
- /* Split remaining MSIX with RDMA after accounting for AEQ MSIX
- */
- pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 +
- ICE_RDMA_NUM_AEQ_MSIX;
- pf->num_lan_msix = v_remain - pf->num_rdma_msix;
- }
-}
-
-/**
- * ice_ena_msix_range - Request a range of MSIX vectors from the OS
- * @pf: board private structure
- *
- * Compute the number of MSIX vectors wanted and request from the OS. Adjust
- * device usage if there are not enough vectors. Return the number of vectors
- * reserved or negative on failure.
- */
-static int ice_ena_msix_range(struct ice_pf *pf)
-{
- int num_cpus, hw_num_msix, v_other, v_wanted, v_actual;
- struct device *dev = ice_pf_to_dev(pf);
- int err;
-
- hw_num_msix = pf->hw.func_caps.common_cap.num_msix_vectors;
- num_cpus = num_online_cpus();
-
- /* LAN miscellaneous handler */
- v_other = ICE_MIN_LAN_OICR_MSIX;
-
- /* Flow Director */
- if (test_bit(ICE_FLAG_FD_ENA, pf->flags))
- v_other += ICE_FDIR_MSIX;
-
- /* switchdev */
- v_other += ICE_ESWITCH_MSIX;
-
- v_wanted = v_other;
-
- /* LAN traffic */
- pf->num_lan_msix = num_cpus;
- v_wanted += pf->num_lan_msix;
-
- /* RDMA auxiliary driver */
- if (ice_is_rdma_ena(pf)) {
- pf->num_rdma_msix = num_cpus + ICE_RDMA_NUM_AEQ_MSIX;
- v_wanted += pf->num_rdma_msix;
- }
-
- if (v_wanted > hw_num_msix) {
- int v_remain;
-
- dev_warn(dev, "not enough device MSI-X vectors. wanted = %d, available = %d\n",
- v_wanted, hw_num_msix);
-
- if (hw_num_msix < ICE_MIN_MSIX) {
- err = -ERANGE;
- goto exit_err;
- }
-
- v_remain = hw_num_msix - v_other;
- if (v_remain < ICE_MIN_LAN_TXRX_MSIX) {
- v_other = ICE_MIN_MSIX - ICE_MIN_LAN_TXRX_MSIX;
- v_remain = ICE_MIN_LAN_TXRX_MSIX;
- }
-
- ice_reduce_msix_usage(pf, v_remain);
- v_wanted = pf->num_lan_msix + pf->num_rdma_msix + v_other;
-
- dev_notice(dev, "Reducing request to %d MSI-X vectors for LAN traffic.\n",
- pf->num_lan_msix);
- if (ice_is_rdma_ena(pf))
- dev_notice(dev, "Reducing request to %d MSI-X vectors for RDMA.\n",
- pf->num_rdma_msix);
- }
-
- /* actually reserve the vectors */
- v_actual = pci_alloc_irq_vectors(pf->pdev, ICE_MIN_MSIX, v_wanted,
- PCI_IRQ_MSIX);
- if (v_actual < 0) {
- dev_err(dev, "unable to reserve MSI-X vectors\n");
- err = v_actual;
- goto exit_err;
- }
-
- if (v_actual < v_wanted) {
- dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n",
- v_wanted, v_actual);
-
- if (v_actual < ICE_MIN_MSIX) {
- /* error if we can't get minimum vectors */
- pci_free_irq_vectors(pf->pdev);
- err = -ERANGE;
- goto exit_err;
- } else {
- int v_remain = v_actual - v_other;
-
- if (v_remain < ICE_MIN_LAN_TXRX_MSIX)
- v_remain = ICE_MIN_LAN_TXRX_MSIX;
-
- ice_reduce_msix_usage(pf, v_remain);
-
- dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n",
- pf->num_lan_msix);
-
- if (ice_is_rdma_ena(pf))
- dev_notice(dev, "Enabled %d MSI-X vectors for RDMA.\n",
- pf->num_rdma_msix);
- }
- }
-
- return v_actual;
-
-exit_err:
- pf->num_rdma_msix = 0;
- pf->num_lan_msix = 0;
- return err;
-}
-
/**
* ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme
* @pf: board private structure
@@ -260,15 +109,18 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
if (!pf->msix.max)
pf->msix.max = total_vectors / 2;
- vectors = ice_ena_msix_range(pf);
-
- if (vectors < 0)
- return -ENOMEM;
-
- if (pci_msix_can_alloc_dyn(pf->pdev))
+ if (pci_msix_can_alloc_dyn(pf->pdev)) {
+ vectors = pf->msix.min;
max_vectors = total_vectors;
- else
+ } else {
+ vectors = pf->msix.max;
max_vectors = vectors;
+ }
+
+ vectors = pci_alloc_irq_vectors(pf->pdev, pf->msix.min, vectors,
+ PCI_IRQ_MSIX);
+ if (vectors < pf->msix.min)
+ return -ENOMEM;
ice_init_irq_tracker(pf, max_vectors, vectors);
--
2.42.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [iwl-next v1 3/7] ice: get rid of num_lan_msix field
2024-02-13 7:35 [iwl-next v1 0/7] ice: managing MSI-X in driver Michal Swiatkowski
2024-02-13 7:35 ` [iwl-next v1 1/7] ice: devlink PF MSI-X max and min parameter Michal Swiatkowski
2024-02-13 7:35 ` [iwl-next v1 2/7] ice: remove splitting MSI-X between features Michal Swiatkowski
@ 2024-02-13 7:35 ` Michal Swiatkowski
2024-02-13 7:35 ` [iwl-next v1 4/7] ice, irdma: move interrupts code to irdma Michal Swiatkowski
` (3 subsequent siblings)
6 siblings, 0 replies; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 7:35 UTC (permalink / raw)
To: intel-wired-lan
Cc: netdev, pawel.chmielewski, sridhar.samudrala, jacob.e.keller,
pio.raczynski, konrad.knitter, marcin.szycik, wojciech.drewek,
nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel, Michal Swiatkowski
Remove the field to allow having more queues than MSI-X on VSI. As
default the number will be the same, but if there won't be more MSI-X
available VSI can run with at least one MSI-X.
Reviewed-by: Wojciech Drewek <wojciech.drewek@intel.com>
Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
---
drivers/net/ethernet/intel/ice/ice.h | 1 -
drivers/net/ethernet/intel/ice/ice_base.c | 10 +++-----
drivers/net/ethernet/intel/ice/ice_ethtool.c | 8 +++---
drivers/net/ethernet/intel/ice/ice_irq.c | 11 +++------
drivers/net/ethernet/intel/ice/ice_lib.c | 26 +++++++++++---------
5 files changed, 27 insertions(+), 29 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 24085f3c0966..5c9a95883456 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -612,7 +612,6 @@ struct ice_pf {
u16 max_pf_txqs; /* Total Tx queues PF wide */
u16 max_pf_rxqs; /* Total Rx queues PF wide */
struct ice_pf_msix msix;
- u16 num_lan_msix; /* Total MSIX vectors for base driver */
u16 num_lan_tx; /* num LAN Tx queues setup */
u16 num_lan_rx; /* num LAN Rx queues setup */
u16 next_vsi; /* Next free slot in pf->vsi[] - 0-based! */
diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c
index 79485c944c9d..d071ead71796 100644
--- a/drivers/net/ethernet/intel/ice/ice_base.c
+++ b/drivers/net/ethernet/intel/ice/ice_base.c
@@ -777,13 +777,11 @@ int ice_vsi_alloc_q_vectors(struct ice_vsi *vsi)
return 0;
err_out:
- while (v_idx--)
- ice_free_q_vector(vsi, v_idx);
- dev_err(dev, "Failed to allocate %d q_vector for VSI %d, ret=%d\n",
- vsi->num_q_vectors, vsi->vsi_num, err);
- vsi->num_q_vectors = 0;
- return err;
+ dev_info(dev, "Failed to allocate %d q_vectors for VSI %d, new value %d",
+ vsi->num_q_vectors, vsi->vsi_num, v_idx);
+ vsi->num_q_vectors = v_idx;
+ return v_idx ? 0 : err;
}
/**
diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c
index 3cc364a4d682..a3e8c15d073d 100644
--- a/drivers/net/ethernet/intel/ice/ice_ethtool.c
+++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c
@@ -3386,8 +3386,8 @@ ice_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info)
*/
static int ice_get_max_txq(struct ice_pf *pf)
{
- return min3(pf->num_lan_msix, (u16)num_online_cpus(),
- (u16)pf->hw.func_caps.common_cap.num_txq);
+ return min_t(u16, num_online_cpus(),
+ pf->hw.func_caps.common_cap.num_txq);
}
/**
@@ -3396,8 +3396,8 @@ static int ice_get_max_txq(struct ice_pf *pf)
*/
static int ice_get_max_rxq(struct ice_pf *pf)
{
- return min3(pf->num_lan_msix, (u16)num_online_cpus(),
- (u16)pf->hw.func_caps.common_cap.num_rxq);
+ return min_t(u16, num_online_cpus(),
+ pf->hw.func_caps.common_cap.num_rxq);
}
/**
diff --git a/drivers/net/ethernet/intel/ice/ice_irq.c b/drivers/net/ethernet/intel/ice/ice_irq.c
index 8116d1d2ffa0..1d7a807454db 100644
--- a/drivers/net/ethernet/intel/ice/ice_irq.c
+++ b/drivers/net/ethernet/intel/ice/ice_irq.c
@@ -101,7 +101,7 @@ void ice_clear_interrupt_scheme(struct ice_pf *pf)
int ice_init_interrupt_scheme(struct ice_pf *pf)
{
int total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
- int vectors, max_vectors;
+ int vectors;
/* load default PF MSI-X range */
if (!pf->msix.min)
@@ -109,20 +109,17 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
if (!pf->msix.max)
pf->msix.max = total_vectors / 2;
- if (pci_msix_can_alloc_dyn(pf->pdev)) {
+ if (pci_msix_can_alloc_dyn(pf->pdev))
vectors = pf->msix.min;
- max_vectors = total_vectors;
- } else {
+ else
vectors = pf->msix.max;
- max_vectors = vectors;
- }
vectors = pci_alloc_irq_vectors(pf->pdev, pf->msix.min, vectors,
PCI_IRQ_MSIX);
if (vectors < pf->msix.min)
return -ENOMEM;
- ice_init_irq_tracker(pf, max_vectors, vectors);
+ ice_init_irq_tracker(pf, pf->msix.max, vectors);
return 0;
}
diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
index 718cb8df7853..445c54b265b6 100644
--- a/drivers/net/ethernet/intel/ice/ice_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_lib.c
@@ -160,6 +160,16 @@ static void ice_vsi_set_num_desc(struct ice_vsi *vsi)
}
}
+static u16 ice_get_rxq_count(struct ice_pf *pf)
+{
+ return min_t(u16, ice_get_avail_rxq_count(pf), num_online_cpus());
+}
+
+static u16 ice_get_txq_count(struct ice_pf *pf)
+{
+ return min_t(u16, ice_get_avail_txq_count(pf), num_online_cpus());
+}
+
/**
* ice_vsi_set_num_qs - Set number of queues, descriptors and vectors for a VSI
* @vsi: the VSI being configured
@@ -181,9 +191,7 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi)
vsi->alloc_txq = vsi->req_txq;
vsi->num_txq = vsi->req_txq;
} else {
- vsi->alloc_txq = min3(pf->num_lan_msix,
- ice_get_avail_txq_count(pf),
- (u16)num_online_cpus());
+ vsi->alloc_txq = ice_get_txq_count(pf);
}
pf->num_lan_tx = vsi->alloc_txq;
@@ -196,17 +204,14 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi)
vsi->alloc_rxq = vsi->req_rxq;
vsi->num_rxq = vsi->req_rxq;
} else {
- vsi->alloc_rxq = min3(pf->num_lan_msix,
- ice_get_avail_rxq_count(pf),
- (u16)num_online_cpus());
+ vsi->alloc_rxq = ice_get_rxq_count(pf);
}
}
pf->num_lan_rx = vsi->alloc_rxq;
- vsi->num_q_vectors = min_t(int, pf->num_lan_msix,
- max_t(int, vsi->alloc_rxq,
- vsi->alloc_txq));
+ vsi->num_q_vectors = max_t(int, vsi->alloc_rxq,
+ vsi->alloc_txq);
break;
case ICE_VSI_VF:
if (vf->num_req_qs)
@@ -1162,12 +1167,11 @@ static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi)
static void
ice_chnl_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt)
{
- struct ice_pf *pf = vsi->back;
u16 qcount, qmap;
u8 offset = 0;
int pow;
- qcount = min_t(int, vsi->num_rxq, pf->num_lan_msix);
+ qcount = vsi->num_rxq;
pow = order_base_2(qcount);
qmap = FIELD_PREP(ICE_AQ_VSI_TC_Q_OFFSET_M, offset);
--
2.42.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [iwl-next v1 4/7] ice, irdma: move interrupts code to irdma
2024-02-13 7:35 [iwl-next v1 0/7] ice: managing MSI-X in driver Michal Swiatkowski
` (2 preceding siblings ...)
2024-02-13 7:35 ` [iwl-next v1 3/7] ice: get rid of num_lan_msix field Michal Swiatkowski
@ 2024-02-13 7:35 ` Michal Swiatkowski
2024-02-13 7:35 ` [iwl-next v1 5/7] ice: treat dyn_allowed only as suggestion Michal Swiatkowski
` (2 subsequent siblings)
6 siblings, 0 replies; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 7:35 UTC (permalink / raw)
To: intel-wired-lan
Cc: netdev, pawel.chmielewski, sridhar.samudrala, jacob.e.keller,
pio.raczynski, konrad.knitter, marcin.szycik, wojciech.drewek,
nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel, Michal Swiatkowski
Move responsibility of MSI-X requesting for RDMA feature from ice driver
to irdma driver. It is done to allow simple fallback when there is not
enough MSI-X available.
Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
---
drivers/infiniband/hw/irdma/hw.c | 2 -
drivers/infiniband/hw/irdma/main.c | 46 ++++++++++++++++-
drivers/infiniband/hw/irdma/main.h | 3 ++
drivers/net/ethernet/intel/ice/ice.h | 2 -
drivers/net/ethernet/intel/ice/ice_idc.c | 64 ++++++------------------
include/linux/net/intel/iidc.h | 2 +
6 files changed, 63 insertions(+), 56 deletions(-)
diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
index bd4b2b896444..748abddf71ef 100644
--- a/drivers/infiniband/hw/irdma/hw.c
+++ b/drivers/infiniband/hw/irdma/hw.c
@@ -497,8 +497,6 @@ static int irdma_save_msix_info(struct irdma_pci_f *rf)
iw_qvlist->num_vectors = rf->msix_count;
if (rf->msix_count <= num_online_cpus())
rf->msix_shared = true;
- else if (rf->msix_count > num_online_cpus() + 1)
- rf->msix_count = num_online_cpus() + 1;
pmsix = rf->msix_entries;
for (i = 0, ceq_idx = 0; i < rf->msix_count; i++, iw_qvinfo++) {
diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c
index 3f13200ff71b..69ad137be7aa 100644
--- a/drivers/infiniband/hw/irdma/main.c
+++ b/drivers/infiniband/hw/irdma/main.c
@@ -206,6 +206,43 @@ static void irdma_lan_unregister_qset(struct irdma_sc_vsi *vsi,
ibdev_dbg(&iwdev->ibdev, "WS: LAN free_res for rdma qset failed.\n");
}
+static int irdma_init_interrupts(struct irdma_pci_f *rf, struct ice_pf *pf)
+{
+ int i;
+
+ rf->msix_count = num_online_cpus() + IRDMA_NUM_AEQ_MSIX;
+ rf->msix_entries = kcalloc(rf->msix_count, sizeof(*rf->msix_entries),
+ GFP_KERNEL);
+ if (!rf->msix_entries)
+ return -ENOMEM;
+
+ for (i = 0; i < rf->msix_count; i++)
+ if (ice_alloc_rdma_qvector(pf, &rf->msix_entries[i]))
+ break;
+
+ if (i < IRDMA_MIN_MSIX) {
+ for (; i >= 0; i--)
+ ice_free_rdma_qvector(pf, &rf->msix_entries[i]);
+
+ kfree(rf->msix_entries);
+ return -ENOMEM;
+ }
+
+ rf->msix_count = i;
+
+ return 0;
+}
+
+static void irdma_deinit_interrupts(struct irdma_pci_f *rf, struct ice_pf *pf)
+{
+ int i;
+
+ for (i = 0; i < rf->msix_count; i++)
+ ice_free_rdma_qvector(pf, &rf->msix_entries[i]);
+
+ kfree(rf->msix_entries);
+}
+
static void irdma_remove(struct auxiliary_device *aux_dev)
{
struct iidc_auxiliary_dev *iidc_adev = container_of(aux_dev,
@@ -216,6 +253,7 @@ static void irdma_remove(struct auxiliary_device *aux_dev)
irdma_ib_unregister_device(iwdev);
ice_rdma_update_vsi_filter(pf, iwdev->vsi_num, false);
+ irdma_deinit_interrupts(iwdev->rf, pf);
pr_debug("INIT: Gen2 PF[%d] device remove success\n", PCI_FUNC(pf->pdev->devfn));
}
@@ -230,9 +268,7 @@ static void irdma_fill_device_info(struct irdma_device *iwdev, struct ice_pf *pf
rf->gen_ops.unregister_qset = irdma_lan_unregister_qset;
rf->hw.hw_addr = pf->hw.hw_addr;
rf->pcidev = pf->pdev;
- rf->msix_count = pf->num_rdma_msix;
rf->pf_id = pf->hw.pf_id;
- rf->msix_entries = &pf->msix_entries[pf->rdma_base_vector];
rf->default_vsi.vsi_idx = vsi->vsi_num;
rf->protocol_used = pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2 ?
IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY;
@@ -281,6 +317,10 @@ static int irdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_
irdma_fill_device_info(iwdev, pf, vsi);
rf = iwdev->rf;
+ err = irdma_init_interrupts(rf, pf);
+ if (err)
+ goto err_init_interrupts;
+
err = irdma_ctrl_init_hw(rf);
if (err)
goto err_ctrl_init;
@@ -311,6 +351,8 @@ static int irdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_
err_rt_init:
irdma_ctrl_deinit_hw(rf);
err_ctrl_init:
+ irdma_deinit_interrupts(rf, pf);
+err_init_interrupts:
kfree(iwdev->rf);
ib_dealloc_device(&iwdev->ibdev);
diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h
index b65bc2ea542f..6c6bafb7b478 100644
--- a/drivers/infiniband/hw/irdma/main.h
+++ b/drivers/infiniband/hw/irdma/main.h
@@ -117,6 +117,9 @@ extern struct auxiliary_driver i40iw_auxiliary_drv;
#define IRDMA_IRQ_NAME_STR_LEN (64)
+#define IRDMA_NUM_AEQ_MSIX 1
+#define IRDMA_MIN_MSIX 2
+
enum init_completion_state {
INVALID_STATE = 0,
INITIAL_STATE,
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 5c9a95883456..1f4d31549528 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -96,8 +96,6 @@
#define ICE_MIN_MSIX (ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_LAN_OICR_MSIX)
#define ICE_MAX_MSIX 256
#define ICE_FDIR_MSIX 2
-#define ICE_RDMA_NUM_AEQ_MSIX 4
-#define ICE_MIN_RDMA_MSIX 2
#define ICE_ESWITCH_MSIX 1
#define ICE_NO_VSI 0xffff
#define ICE_VSI_MAP_CONTIG 0
diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c
index 145b27f2a4ce..bab3e81cad5d 100644
--- a/drivers/net/ethernet/intel/ice/ice_idc.c
+++ b/drivers/net/ethernet/intel/ice/ice_idc.c
@@ -228,61 +228,34 @@ void ice_get_qos_params(struct ice_pf *pf, struct iidc_qos_params *qos)
}
EXPORT_SYMBOL_GPL(ice_get_qos_params);
-/**
- * ice_alloc_rdma_qvectors - Allocate vector resources for RDMA driver
- * @pf: board private structure to initialize
- */
-static int ice_alloc_rdma_qvectors(struct ice_pf *pf)
+int ice_alloc_rdma_qvector(struct ice_pf *pf, struct msix_entry *entry)
{
- if (ice_is_rdma_ena(pf)) {
- int i;
-
- pf->msix_entries = kcalloc(pf->num_rdma_msix,
- sizeof(*pf->msix_entries),
- GFP_KERNEL);
- if (!pf->msix_entries)
- return -ENOMEM;
+ struct msi_map map = ice_alloc_irq(pf, true);
- /* RDMA is the only user of pf->msix_entries array */
- pf->rdma_base_vector = 0;
-
- for (i = 0; i < pf->num_rdma_msix; i++) {
- struct msix_entry *entry = &pf->msix_entries[i];
- struct msi_map map;
+ if (map.index < 0)
+ return -ENOMEM;
- map = ice_alloc_irq(pf, false);
- if (map.index < 0)
- break;
+ entry->entry = map.index;
+ entry->vector = map.virq;
- entry->entry = map.index;
- entry->vector = map.virq;
- }
- }
return 0;
}
+EXPORT_SYMBOL_GPL(ice_alloc_rdma_qvector);
/**
* ice_free_rdma_qvector - free vector resources reserved for RDMA driver
* @pf: board private structure to initialize
+ * @entry: MSI-X entry to be removed
*/
-static void ice_free_rdma_qvector(struct ice_pf *pf)
+void ice_free_rdma_qvector(struct ice_pf *pf, struct msix_entry *entry)
{
- int i;
-
- if (!pf->msix_entries)
- return;
-
- for (i = 0; i < pf->num_rdma_msix; i++) {
- struct msi_map map;
+ struct msi_map map;
- map.index = pf->msix_entries[i].entry;
- map.virq = pf->msix_entries[i].vector;
- ice_free_irq(pf, map);
- }
-
- kfree(pf->msix_entries);
- pf->msix_entries = NULL;
+ map.index = entry->entry;
+ map.virq = entry->vector;
+ ice_free_irq(pf, map);
}
+EXPORT_SYMBOL_GPL(ice_free_rdma_qvector);
/**
* ice_adev_release - function to be mapped to AUX dev's release op
@@ -382,12 +355,6 @@ int ice_init_rdma(struct ice_pf *pf)
return -ENOMEM;
}
- /* Reserve vector resources */
- ret = ice_alloc_rdma_qvectors(pf);
- if (ret < 0) {
- dev_err(dev, "failed to reserve vectors for RDMA\n");
- goto err_reserve_rdma_qvector;
- }
pf->rdma_mode |= IIDC_RDMA_PROTOCOL_ROCEV2;
ret = ice_plug_aux_dev(pf);
if (ret)
@@ -395,8 +362,6 @@ int ice_init_rdma(struct ice_pf *pf)
return 0;
err_plug_aux_dev:
- ice_free_rdma_qvector(pf);
-err_reserve_rdma_qvector:
pf->adev = NULL;
xa_erase(&ice_aux_id, pf->aux_idx);
return ret;
@@ -412,6 +377,5 @@ void ice_deinit_rdma(struct ice_pf *pf)
return;
ice_unplug_aux_dev(pf);
- ice_free_rdma_qvector(pf);
xa_erase(&ice_aux_id, pf->aux_idx);
}
diff --git a/include/linux/net/intel/iidc.h b/include/linux/net/intel/iidc.h
index 1c1332e4df26..13274c3def66 100644
--- a/include/linux/net/intel/iidc.h
+++ b/include/linux/net/intel/iidc.h
@@ -78,6 +78,8 @@ int ice_del_rdma_qset(struct ice_pf *pf, struct iidc_rdma_qset_params *qset);
int ice_rdma_request_reset(struct ice_pf *pf, enum iidc_reset_type reset_type);
int ice_rdma_update_vsi_filter(struct ice_pf *pf, u16 vsi_id, bool enable);
void ice_get_qos_params(struct ice_pf *pf, struct iidc_qos_params *qos);
+int ice_alloc_rdma_qvector(struct ice_pf *pf, struct msix_entry *entry);
+void ice_free_rdma_qvector(struct ice_pf *pf, struct msix_entry *entry);
/* Structure representing auxiliary driver tailored information about the core
* PCI dev, each auxiliary driver using the IIDC interface will have an
--
2.42.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [iwl-next v1 5/7] ice: treat dyn_allowed only as suggestion
2024-02-13 7:35 [iwl-next v1 0/7] ice: managing MSI-X in driver Michal Swiatkowski
` (3 preceding siblings ...)
2024-02-13 7:35 ` [iwl-next v1 4/7] ice, irdma: move interrupts code to irdma Michal Swiatkowski
@ 2024-02-13 7:35 ` Michal Swiatkowski
2024-02-13 7:35 ` [iwl-next v1 6/7] ice: enable_rdma devlink param Michal Swiatkowski
2024-02-13 7:35 ` [iwl-next v1 7/7] ice: simplify VF MSI-X managing Michal Swiatkowski
6 siblings, 0 replies; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 7:35 UTC (permalink / raw)
To: intel-wired-lan
Cc: netdev, pawel.chmielewski, sridhar.samudrala, jacob.e.keller,
pio.raczynski, konrad.knitter, marcin.szycik, wojciech.drewek,
nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel, Michal Swiatkowski
It can be needed to have some MSI-X allocated as static and rest as
dynamic. For example on PF VSI. We want to always have minimum one MSI-X
on it, because of that it is allocated as a static one, rest can be
dynamic if it is supported.
Change the ice_get_irq_res() to allow using static entries if they are
free even if caller wants dynamic one.
Adjust limit values to the new approach. Min and max in limit means the
values that are valid, so decrease max and num_static by one.
Set vsi::irq_dyn_alloc if dynamic allocation is supported.
Reviewed-by: Wojciech Drewek <wojciech.drewek@intel.com>
Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
---
drivers/net/ethernet/intel/ice/ice_irq.c | 25 ++++++++++++------------
drivers/net/ethernet/intel/ice/ice_lib.c | 2 ++
2 files changed, 15 insertions(+), 12 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_irq.c b/drivers/net/ethernet/intel/ice/ice_irq.c
index 1d7a807454db..f95f6e5987b7 100644
--- a/drivers/net/ethernet/intel/ice/ice_irq.c
+++ b/drivers/net/ethernet/intel/ice/ice_irq.c
@@ -45,7 +45,7 @@ static void ice_free_irq_res(struct ice_pf *pf, u16 index)
/**
* ice_get_irq_res - get an interrupt resource
* @pf: board private structure
- * @dyn_only: force entry to be dynamically allocated
+ * @dyn_allowed: allow entry to be dynamically allocated
*
* Allocate new irq entry in the free slot of the tracker. Since xarray
* is used, always allocate new entry at the lowest possible index. Set
@@ -53,11 +53,12 @@ static void ice_free_irq_res(struct ice_pf *pf, u16 index)
*
* Returns allocated irq entry or NULL on failure.
*/
-static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only)
+static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf,
+ bool dyn_allowed)
{
- struct xa_limit limit = { .max = pf->irq_tracker.num_entries,
+ struct xa_limit limit = { .max = pf->irq_tracker.num_entries - 1,
.min = 0 };
- unsigned int num_static = pf->irq_tracker.num_static;
+ unsigned int num_static = pf->irq_tracker.num_static - 1;
struct ice_irq_entry *entry;
unsigned int index;
int ret;
@@ -66,9 +67,9 @@ static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only)
if (!entry)
return NULL;
- /* skip preallocated entries if the caller says so */
- if (dyn_only)
- limit.min = num_static;
+ /* only already allocated if the caller says so */
+ if (!dyn_allowed)
+ limit.max = num_static;
ret = xa_alloc(&pf->irq_tracker.entries, &index, entry, limit,
GFP_KERNEL);
@@ -78,7 +79,7 @@ static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only)
entry = NULL;
} else {
entry->index = index;
- entry->dynamic = index >= num_static;
+ entry->dynamic = index > num_static;
}
return entry;
@@ -127,7 +128,7 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
/**
* ice_alloc_irq - Allocate new interrupt vector
* @pf: board private structure
- * @dyn_only: force dynamic allocation of the interrupt
+ * @dyn_allowed: allow dynamic allocation of the interrupt
*
* Allocate new interrupt vector for a given owner id.
* return struct msi_map with interrupt details and track
@@ -140,20 +141,20 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
* interrupt will be allocated with pci_msix_alloc_irq_at.
*
* Some callers may only support dynamically allocated interrupts.
- * This is indicated with dyn_only flag.
+ * This is indicated with dyn_allowed flag.
*
* On failure, return map with negative .index. The caller
* is expected to check returned map index.
*
*/
-struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_only)
+struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_allowed)
{
int sriov_base_vector = pf->sriov_base_vector;
struct msi_map map = { .index = -ENOENT };
struct device *dev = ice_pf_to_dev(pf);
struct ice_irq_entry *entry;
- entry = ice_get_irq_res(pf, dyn_only);
+ entry = ice_get_irq_res(pf, dyn_allowed);
if (!entry)
return map;
diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
index 445c54b265b6..a30d2d2b51c1 100644
--- a/drivers/net/ethernet/intel/ice/ice_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_lib.c
@@ -570,6 +570,8 @@ ice_vsi_alloc_def(struct ice_vsi *vsi, struct ice_channel *ch)
return -ENOMEM;
}
+ vsi->irq_dyn_alloc = pci_msix_can_alloc_dyn(vsi->back->pdev);
+
switch (vsi->type) {
case ICE_VSI_PF:
/* Setup default MSIX irq handler for VSI */
--
2.42.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* [iwl-next v1 6/7] ice: enable_rdma devlink param
2024-02-13 7:35 [iwl-next v1 0/7] ice: managing MSI-X in driver Michal Swiatkowski
` (4 preceding siblings ...)
2024-02-13 7:35 ` [iwl-next v1 5/7] ice: treat dyn_allowed only as suggestion Michal Swiatkowski
@ 2024-02-13 7:35 ` Michal Swiatkowski
2024-02-13 9:07 ` Jiri Pirko
2024-02-13 7:35 ` [iwl-next v1 7/7] ice: simplify VF MSI-X managing Michal Swiatkowski
6 siblings, 1 reply; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 7:35 UTC (permalink / raw)
To: intel-wired-lan
Cc: netdev, pawel.chmielewski, sridhar.samudrala, jacob.e.keller,
pio.raczynski, konrad.knitter, marcin.szycik, wojciech.drewek,
nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel, Michal Swiatkowski,
Jan Sokolowski
Implement enable_rdma devlink parameter to allow user to turn RDMA
feature on and off.
It is useful when there is no enough interrupts and user doesn't need
RDMA feature.
Reviewed-by: Jan Sokolowski <jan.sokolowski@intel.com>
Reviewed-by: Przemek Kitszel <przemyslaw.kitszel@intel.com>
Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
---
drivers/net/ethernet/intel/ice/ice_devlink.c | 32 ++++++++++++++++++--
drivers/net/ethernet/intel/ice/ice_lib.c | 8 ++++-
drivers/net/ethernet/intel/ice/ice_main.c | 18 +++++------
3 files changed, 45 insertions(+), 13 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
index b82ff9556a4b..4f048268db72 100644
--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
@@ -1675,6 +1675,19 @@ ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
return 0;
}
+static int ice_devlink_enable_rdma_validate(struct devlink *devlink, u32 id,
+ union devlink_param_value val,
+ struct netlink_ext_ack *extack)
+{
+ struct ice_pf *pf = devlink_priv(devlink);
+ bool new_state = val.vbool;
+
+ if (new_state && !test_bit(ICE_FLAG_RDMA_ENA, pf->flags))
+ return -EOPNOTSUPP;
+
+ return 0;
+}
+
static const struct devlink_param ice_devlink_params[] = {
DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
ice_devlink_enable_roce_get,
@@ -1700,6 +1713,8 @@ static const struct devlink_param ice_devlink_params[] = {
ice_devlink_msix_min_pf_get,
ice_devlink_msix_min_pf_set,
ice_devlink_msix_min_pf_validate),
+ DEVLINK_PARAM_GENERIC(ENABLE_RDMA, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
+ NULL, NULL, ice_devlink_enable_rdma_validate),
};
static void ice_devlink_free(void *devlink_ptr)
@@ -1776,9 +1791,22 @@ ice_devlink_set_switch_id(struct ice_pf *pf, struct netdev_phys_item_id *ppid)
int ice_devlink_register_params(struct ice_pf *pf)
{
struct devlink *devlink = priv_to_devlink(pf);
+ union devlink_param_value value;
+ int err;
+
+ err = devlink_params_register(devlink, ice_devlink_params,
+ ARRAY_SIZE(ice_devlink_params));
+ if (err)
+ return err;
- return devlink_params_register(devlink, ice_devlink_params,
- ARRAY_SIZE(ice_devlink_params));
+ devl_lock(devlink);
+ value.vbool = test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
+ devl_param_driverinit_value_set(devlink,
+ DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
+ value);
+ devl_unlock(devlink);
+
+ return 0;
}
void ice_devlink_unregister_params(struct ice_pf *pf)
diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
index a30d2d2b51c1..4d5c3d699044 100644
--- a/drivers/net/ethernet/intel/ice/ice_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_lib.c
@@ -829,7 +829,13 @@ bool ice_is_safe_mode(struct ice_pf *pf)
*/
bool ice_is_rdma_ena(struct ice_pf *pf)
{
- return test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
+ union devlink_param_value value;
+ int err;
+
+ err = devl_param_driverinit_value_get(priv_to_devlink(pf),
+ DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
+ &value);
+ return err ? false : value.vbool;
}
/**
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index 824732f16112..4dd59d888ec7 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -5177,23 +5177,21 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
if (err)
goto err_init;
+ err = ice_init_devlink(pf);
+ if (err)
+ goto err_init_devlink;
+
devl_lock(priv_to_devlink(pf));
err = ice_load(pf);
devl_unlock(priv_to_devlink(pf));
if (err)
goto err_load;
- err = ice_init_devlink(pf);
- if (err)
- goto err_init_devlink;
-
return 0;
-err_init_devlink:
- devl_lock(priv_to_devlink(pf));
- ice_unload(pf);
- devl_unlock(priv_to_devlink(pf));
err_load:
+ ice_deinit_devlink(pf);
+err_init_devlink:
ice_deinit(pf);
err_init:
pci_disable_device(pdev);
@@ -5290,12 +5288,12 @@ static void ice_remove(struct pci_dev *pdev)
if (!ice_is_safe_mode(pf))
ice_remove_arfs(pf);
- ice_deinit_devlink(pf);
-
devl_lock(priv_to_devlink(pf));
ice_unload(pf);
devl_unlock(priv_to_devlink(pf));
+ ice_deinit_devlink(pf);
+
ice_deinit(pf);
ice_vsi_release_all(pf);
--
2.42.0
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [iwl-next v1 6/7] ice: enable_rdma devlink param
2024-02-13 7:35 ` [iwl-next v1 6/7] ice: enable_rdma devlink param Michal Swiatkowski
@ 2024-02-13 9:07 ` Jiri Pirko
2024-02-13 9:58 ` Michal Swiatkowski
0 siblings, 1 reply; 17+ messages in thread
From: Jiri Pirko @ 2024-02-13 9:07 UTC (permalink / raw)
To: Michal Swiatkowski
Cc: intel-wired-lan, netdev, pawel.chmielewski, sridhar.samudrala,
jacob.e.keller, pio.raczynski, konrad.knitter, marcin.szycik,
wojciech.drewek, nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel,
Jan Sokolowski
Tue, Feb 13, 2024 at 08:35:08AM CET, michal.swiatkowski@linux.intel.com wrote:
>Implement enable_rdma devlink parameter to allow user to turn RDMA
>feature on and off.
>
>It is useful when there is no enough interrupts and user doesn't need
>RDMA feature.
>
>Reviewed-by: Jan Sokolowski <jan.sokolowski@intel.com>
>Reviewed-by: Przemek Kitszel <przemyslaw.kitszel@intel.com>
>Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
>---
> drivers/net/ethernet/intel/ice/ice_devlink.c | 32 ++++++++++++++++++--
> drivers/net/ethernet/intel/ice/ice_lib.c | 8 ++++-
> drivers/net/ethernet/intel/ice/ice_main.c | 18 +++++------
> 3 files changed, 45 insertions(+), 13 deletions(-)
>
>diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
>index b82ff9556a4b..4f048268db72 100644
>--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
>+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
>@@ -1675,6 +1675,19 @@ ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
> return 0;
> }
>
>+static int ice_devlink_enable_rdma_validate(struct devlink *devlink, u32 id,
>+ union devlink_param_value val,
>+ struct netlink_ext_ack *extack)
>+{
>+ struct ice_pf *pf = devlink_priv(devlink);
>+ bool new_state = val.vbool;
>+
>+ if (new_state && !test_bit(ICE_FLAG_RDMA_ENA, pf->flags))
>+ return -EOPNOTSUPP;
>+
>+ return 0;
>+}
>+
> static const struct devlink_param ice_devlink_params[] = {
> DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
> ice_devlink_enable_roce_get,
>@@ -1700,6 +1713,8 @@ static const struct devlink_param ice_devlink_params[] = {
> ice_devlink_msix_min_pf_get,
> ice_devlink_msix_min_pf_set,
> ice_devlink_msix_min_pf_validate),
>+ DEVLINK_PARAM_GENERIC(ENABLE_RDMA, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
>+ NULL, NULL, ice_devlink_enable_rdma_validate),
> };
>
> static void ice_devlink_free(void *devlink_ptr)
>@@ -1776,9 +1791,22 @@ ice_devlink_set_switch_id(struct ice_pf *pf, struct netdev_phys_item_id *ppid)
> int ice_devlink_register_params(struct ice_pf *pf)
> {
> struct devlink *devlink = priv_to_devlink(pf);
>+ union devlink_param_value value;
>+ int err;
>+
>+ err = devlink_params_register(devlink, ice_devlink_params,
Interesting, can't you just take the lock before this and call
devl_params_register()?
Moreover, could you please fix your init/cleanup paths for hold devlink
instance lock the whole time?
pw-bot: cr
>+ ARRAY_SIZE(ice_devlink_params));
>+ if (err)
>+ return err;
>
>- return devlink_params_register(devlink, ice_devlink_params,
>- ARRAY_SIZE(ice_devlink_params));
>+ devl_lock(devlink);
>+ value.vbool = test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
>+ devl_param_driverinit_value_set(devlink,
>+ DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
>+ value);
>+ devl_unlock(devlink);
>+
>+ return 0;
> }
>
> void ice_devlink_unregister_params(struct ice_pf *pf)
>diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
>index a30d2d2b51c1..4d5c3d699044 100644
>--- a/drivers/net/ethernet/intel/ice/ice_lib.c
>+++ b/drivers/net/ethernet/intel/ice/ice_lib.c
>@@ -829,7 +829,13 @@ bool ice_is_safe_mode(struct ice_pf *pf)
> */
> bool ice_is_rdma_ena(struct ice_pf *pf)
> {
>- return test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
>+ union devlink_param_value value;
>+ int err;
>+
>+ err = devl_param_driverinit_value_get(priv_to_devlink(pf),
>+ DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
>+ &value);
>+ return err ? false : value.vbool;
> }
>
> /**
>diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
>index 824732f16112..4dd59d888ec7 100644
>--- a/drivers/net/ethernet/intel/ice/ice_main.c
>+++ b/drivers/net/ethernet/intel/ice/ice_main.c
>@@ -5177,23 +5177,21 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
> if (err)
> goto err_init;
>
>+ err = ice_init_devlink(pf);
>+ if (err)
>+ goto err_init_devlink;
>+
> devl_lock(priv_to_devlink(pf));
> err = ice_load(pf);
> devl_unlock(priv_to_devlink(pf));
> if (err)
> goto err_load;
>
>- err = ice_init_devlink(pf);
>- if (err)
>- goto err_init_devlink;
>-
> return 0;
>
>-err_init_devlink:
>- devl_lock(priv_to_devlink(pf));
>- ice_unload(pf);
>- devl_unlock(priv_to_devlink(pf));
> err_load:
>+ ice_deinit_devlink(pf);
>+err_init_devlink:
> ice_deinit(pf);
> err_init:
> pci_disable_device(pdev);
>@@ -5290,12 +5288,12 @@ static void ice_remove(struct pci_dev *pdev)
> if (!ice_is_safe_mode(pf))
> ice_remove_arfs(pf);
>
>- ice_deinit_devlink(pf);
>-
> devl_lock(priv_to_devlink(pf));
> ice_unload(pf);
> devl_unlock(priv_to_devlink(pf));
>
>+ ice_deinit_devlink(pf);
>+
> ice_deinit(pf);
> ice_vsi_release_all(pf);
>
>--
>2.42.0
>
>
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [iwl-next v1 6/7] ice: enable_rdma devlink param
2024-02-13 9:07 ` Jiri Pirko
@ 2024-02-13 9:58 ` Michal Swiatkowski
2024-02-13 10:19 ` Wojciech Drewek
0 siblings, 1 reply; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 9:58 UTC (permalink / raw)
To: Jiri Pirko
Cc: intel-wired-lan, netdev, pawel.chmielewski, sridhar.samudrala,
jacob.e.keller, pio.raczynski, konrad.knitter, marcin.szycik,
wojciech.drewek, nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel,
Jan Sokolowski
On Tue, Feb 13, 2024 at 10:07:17AM +0100, Jiri Pirko wrote:
> Tue, Feb 13, 2024 at 08:35:08AM CET, michal.swiatkowski@linux.intel.com wrote:
> >Implement enable_rdma devlink parameter to allow user to turn RDMA
> >feature on and off.
> >
> >It is useful when there is no enough interrupts and user doesn't need
> >RDMA feature.
> >
> >Reviewed-by: Jan Sokolowski <jan.sokolowski@intel.com>
> >Reviewed-by: Przemek Kitszel <przemyslaw.kitszel@intel.com>
> >Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
> >---
> > drivers/net/ethernet/intel/ice/ice_devlink.c | 32 ++++++++++++++++++--
> > drivers/net/ethernet/intel/ice/ice_lib.c | 8 ++++-
> > drivers/net/ethernet/intel/ice/ice_main.c | 18 +++++------
> > 3 files changed, 45 insertions(+), 13 deletions(-)
> >
> >diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
> >index b82ff9556a4b..4f048268db72 100644
> >--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
> >+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
> >@@ -1675,6 +1675,19 @@ ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
> > return 0;
> > }
> >
> >+static int ice_devlink_enable_rdma_validate(struct devlink *devlink, u32 id,
> >+ union devlink_param_value val,
> >+ struct netlink_ext_ack *extack)
> >+{
> >+ struct ice_pf *pf = devlink_priv(devlink);
> >+ bool new_state = val.vbool;
> >+
> >+ if (new_state && !test_bit(ICE_FLAG_RDMA_ENA, pf->flags))
> >+ return -EOPNOTSUPP;
> >+
> >+ return 0;
> >+}
> >+
> > static const struct devlink_param ice_devlink_params[] = {
> > DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
> > ice_devlink_enable_roce_get,
> >@@ -1700,6 +1713,8 @@ static const struct devlink_param ice_devlink_params[] = {
> > ice_devlink_msix_min_pf_get,
> > ice_devlink_msix_min_pf_set,
> > ice_devlink_msix_min_pf_validate),
> >+ DEVLINK_PARAM_GENERIC(ENABLE_RDMA, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
> >+ NULL, NULL, ice_devlink_enable_rdma_validate),
> > };
> >
> > static void ice_devlink_free(void *devlink_ptr)
> >@@ -1776,9 +1791,22 @@ ice_devlink_set_switch_id(struct ice_pf *pf, struct netdev_phys_item_id *ppid)
> > int ice_devlink_register_params(struct ice_pf *pf)
> > {
> > struct devlink *devlink = priv_to_devlink(pf);
> >+ union devlink_param_value value;
> >+ int err;
> >+
> >+ err = devlink_params_register(devlink, ice_devlink_params,
>
> Interesting, can't you just take the lock before this and call
> devl_params_register()?
>
I mess up a locking here and also in subfunction patchset. I will follow
you suggestion and take lock for whole init/cleanup. Thanks.
> Moreover, could you please fix your init/cleanup paths for hold devlink
> instance lock the whole time?
>
You right, I will prepare patch for it.
>
> pw-bot: cr
>
>
> >+ ARRAY_SIZE(ice_devlink_params));
> >+ if (err)
> >+ return err;
> >
> >- return devlink_params_register(devlink, ice_devlink_params,
> >- ARRAY_SIZE(ice_devlink_params));
> >+ devl_lock(devlink);
> >+ value.vbool = test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
> >+ devl_param_driverinit_value_set(devlink,
> >+ DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
> >+ value);
> >+ devl_unlock(devlink);
> >+
> >+ return 0;
> > }
> >
> > void ice_devlink_unregister_params(struct ice_pf *pf)
> >diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
> >index a30d2d2b51c1..4d5c3d699044 100644
> >--- a/drivers/net/ethernet/intel/ice/ice_lib.c
> >+++ b/drivers/net/ethernet/intel/ice/ice_lib.c
> >@@ -829,7 +829,13 @@ bool ice_is_safe_mode(struct ice_pf *pf)
> > */
> > bool ice_is_rdma_ena(struct ice_pf *pf)
> > {
> >- return test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
> >+ union devlink_param_value value;
> >+ int err;
> >+
> >+ err = devl_param_driverinit_value_get(priv_to_devlink(pf),
> >+ DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
> >+ &value);
> >+ return err ? false : value.vbool;
> > }
> >
> > /**
> >diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
> >index 824732f16112..4dd59d888ec7 100644
> >--- a/drivers/net/ethernet/intel/ice/ice_main.c
> >+++ b/drivers/net/ethernet/intel/ice/ice_main.c
> >@@ -5177,23 +5177,21 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
> > if (err)
> > goto err_init;
> >
> >+ err = ice_init_devlink(pf);
> >+ if (err)
> >+ goto err_init_devlink;
> >+
> > devl_lock(priv_to_devlink(pf));
> > err = ice_load(pf);
> > devl_unlock(priv_to_devlink(pf));
> > if (err)
> > goto err_load;
> >
> >- err = ice_init_devlink(pf);
> >- if (err)
> >- goto err_init_devlink;
> >-
> > return 0;
> >
> >-err_init_devlink:
> >- devl_lock(priv_to_devlink(pf));
> >- ice_unload(pf);
> >- devl_unlock(priv_to_devlink(pf));
> > err_load:
> >+ ice_deinit_devlink(pf);
> >+err_init_devlink:
> > ice_deinit(pf);
> > err_init:
> > pci_disable_device(pdev);
> >@@ -5290,12 +5288,12 @@ static void ice_remove(struct pci_dev *pdev)
> > if (!ice_is_safe_mode(pf))
> > ice_remove_arfs(pf);
> >
> >- ice_deinit_devlink(pf);
> >-
> > devl_lock(priv_to_devlink(pf));
> > ice_unload(pf);
> > devl_unlock(priv_to_devlink(pf));
> >
> >+ ice_deinit_devlink(pf);
> >+
> > ice_deinit(pf);
> > ice_vsi_release_all(pf);
> >
> >--
> >2.42.0
> >
> >
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [iwl-next v1 6/7] ice: enable_rdma devlink param
2024-02-13 9:58 ` Michal Swiatkowski
@ 2024-02-13 10:19 ` Wojciech Drewek
2024-02-13 10:48 ` Michal Swiatkowski
2024-02-13 11:35 ` Jiri Pirko
0 siblings, 2 replies; 17+ messages in thread
From: Wojciech Drewek @ 2024-02-13 10:19 UTC (permalink / raw)
To: Michal Swiatkowski, Jiri Pirko
Cc: intel-wired-lan, netdev, pawel.chmielewski, sridhar.samudrala,
jacob.e.keller, pio.raczynski, konrad.knitter, marcin.szycik,
nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel, Jan Sokolowski
On 13.02.2024 10:58, Michal Swiatkowski wrote:
> On Tue, Feb 13, 2024 at 10:07:17AM +0100, Jiri Pirko wrote:
>> Tue, Feb 13, 2024 at 08:35:08AM CET, michal.swiatkowski@linux.intel.com wrote:
>>> Implement enable_rdma devlink parameter to allow user to turn RDMA
>>> feature on and off.
>>>
>>> It is useful when there is no enough interrupts and user doesn't need
>>> RDMA feature.
>>>
>>> Reviewed-by: Jan Sokolowski <jan.sokolowski@intel.com>
>>> Reviewed-by: Przemek Kitszel <przemyslaw.kitszel@intel.com>
>>> Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
>>> ---
>>> drivers/net/ethernet/intel/ice/ice_devlink.c | 32 ++++++++++++++++++--
>>> drivers/net/ethernet/intel/ice/ice_lib.c | 8 ++++-
>>> drivers/net/ethernet/intel/ice/ice_main.c | 18 +++++------
>>> 3 files changed, 45 insertions(+), 13 deletions(-)
>>>
>>> diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
>>> index b82ff9556a4b..4f048268db72 100644
>>> --- a/drivers/net/ethernet/intel/ice/ice_devlink.c
>>> +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
>>> @@ -1675,6 +1675,19 @@ ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
>>> return 0;
>>> }
>>>
>>> +static int ice_devlink_enable_rdma_validate(struct devlink *devlink, u32 id,
>>> + union devlink_param_value val,
>>> + struct netlink_ext_ack *extack)
>>> +{
>>> + struct ice_pf *pf = devlink_priv(devlink);
>>> + bool new_state = val.vbool;
>>> +
>>> + if (new_state && !test_bit(ICE_FLAG_RDMA_ENA, pf->flags))
>>> + return -EOPNOTSUPP;
>>> +
>>> + return 0;
>>> +}
>>> +
>>> static const struct devlink_param ice_devlink_params[] = {
>>> DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
>>> ice_devlink_enable_roce_get,
>>> @@ -1700,6 +1713,8 @@ static const struct devlink_param ice_devlink_params[] = {
>>> ice_devlink_msix_min_pf_get,
>>> ice_devlink_msix_min_pf_set,
>>> ice_devlink_msix_min_pf_validate),
>>> + DEVLINK_PARAM_GENERIC(ENABLE_RDMA, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
>>> + NULL, NULL, ice_devlink_enable_rdma_validate),
>>> };
>>>
>>> static void ice_devlink_free(void *devlink_ptr)
>>> @@ -1776,9 +1791,22 @@ ice_devlink_set_switch_id(struct ice_pf *pf, struct netdev_phys_item_id *ppid)
>>> int ice_devlink_register_params(struct ice_pf *pf)
>>> {
>>> struct devlink *devlink = priv_to_devlink(pf);
>>> + union devlink_param_value value;
>>> + int err;
>>> +
>>> + err = devlink_params_register(devlink, ice_devlink_params,
>>
>> Interesting, can't you just take the lock before this and call
>> devl_params_register()?
>>
> I mess up a locking here and also in subfunction patchset. I will follow
> you suggestion and take lock for whole init/cleanup. Thanks.
>
>> Moreover, could you please fix your init/cleanup paths for hold devlink
>> instance lock the whole time?
>>
> You right, I will prepare patch for it.
I think my patch implements your suggestion Jiri:
https://lore.kernel.org/netdev/20240212211202.1051990-5-anthony.l.nguyen@intel.com/
>
>>
>> pw-bot: cr
>>
>>
>>> + ARRAY_SIZE(ice_devlink_params));
>>> + if (err)
>>> + return err;
>>>
>>> - return devlink_params_register(devlink, ice_devlink_params,
>>> - ARRAY_SIZE(ice_devlink_params));
>>> + devl_lock(devlink);
>>> + value.vbool = test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
>>> + devl_param_driverinit_value_set(devlink,
>>> + DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
>>> + value);
>>> + devl_unlock(devlink);
>>> +
>>> + return 0;
>>> }
>>>
>>> void ice_devlink_unregister_params(struct ice_pf *pf)
>>> diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
>>> index a30d2d2b51c1..4d5c3d699044 100644
>>> --- a/drivers/net/ethernet/intel/ice/ice_lib.c
>>> +++ b/drivers/net/ethernet/intel/ice/ice_lib.c
>>> @@ -829,7 +829,13 @@ bool ice_is_safe_mode(struct ice_pf *pf)
>>> */
>>> bool ice_is_rdma_ena(struct ice_pf *pf)
>>> {
>>> - return test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
>>> + union devlink_param_value value;
>>> + int err;
>>> +
>>> + err = devl_param_driverinit_value_get(priv_to_devlink(pf),
>>> + DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
>>> + &value);
>>> + return err ? false : value.vbool;
>>> }
>>>
>>> /**
>>> diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
>>> index 824732f16112..4dd59d888ec7 100644
>>> --- a/drivers/net/ethernet/intel/ice/ice_main.c
>>> +++ b/drivers/net/ethernet/intel/ice/ice_main.c
>>> @@ -5177,23 +5177,21 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
>>> if (err)
>>> goto err_init;
>>>
>>> + err = ice_init_devlink(pf);
>>> + if (err)
>>> + goto err_init_devlink;
>>> +
>>> devl_lock(priv_to_devlink(pf));
>>> err = ice_load(pf);
>>> devl_unlock(priv_to_devlink(pf));
>>> if (err)
>>> goto err_load;
>>>
>>> - err = ice_init_devlink(pf);
>>> - if (err)
>>> - goto err_init_devlink;
>>> -
>>> return 0;
>>>
>>> -err_init_devlink:
>>> - devl_lock(priv_to_devlink(pf));
>>> - ice_unload(pf);
>>> - devl_unlock(priv_to_devlink(pf));
>>> err_load:
>>> + ice_deinit_devlink(pf);
>>> +err_init_devlink:
>>> ice_deinit(pf);
>>> err_init:
>>> pci_disable_device(pdev);
>>> @@ -5290,12 +5288,12 @@ static void ice_remove(struct pci_dev *pdev)
>>> if (!ice_is_safe_mode(pf))
>>> ice_remove_arfs(pf);
>>>
>>> - ice_deinit_devlink(pf);
>>> -
>>> devl_lock(priv_to_devlink(pf));
>>> ice_unload(pf);
>>> devl_unlock(priv_to_devlink(pf));
>>>
>>> + ice_deinit_devlink(pf);
>>> +
>>> ice_deinit(pf);
>>> ice_vsi_release_all(pf);
>>>
>>> --
>>> 2.42.0
>>>
>>>
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [iwl-next v1 6/7] ice: enable_rdma devlink param
2024-02-13 10:19 ` Wojciech Drewek
@ 2024-02-13 10:48 ` Michal Swiatkowski
2024-02-13 11:35 ` Jiri Pirko
1 sibling, 0 replies; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 10:48 UTC (permalink / raw)
To: Wojciech Drewek
Cc: Jiri Pirko, intel-wired-lan, netdev, pawel.chmielewski,
sridhar.samudrala, jacob.e.keller, pio.raczynski, konrad.knitter,
marcin.szycik, nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel,
Jan Sokolowski
On Tue, Feb 13, 2024 at 11:19:49AM +0100, Wojciech Drewek wrote:
>
>
> On 13.02.2024 10:58, Michal Swiatkowski wrote:
> > On Tue, Feb 13, 2024 at 10:07:17AM +0100, Jiri Pirko wrote:
> >> Tue, Feb 13, 2024 at 08:35:08AM CET, michal.swiatkowski@linux.intel.com wrote:
> >>> Implement enable_rdma devlink parameter to allow user to turn RDMA
> >>> feature on and off.
> >>>
> >>> It is useful when there is no enough interrupts and user doesn't need
> >>> RDMA feature.
> >>>
> >>> Reviewed-by: Jan Sokolowski <jan.sokolowski@intel.com>
> >>> Reviewed-by: Przemek Kitszel <przemyslaw.kitszel@intel.com>
> >>> Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
> >>> ---
> >>> drivers/net/ethernet/intel/ice/ice_devlink.c | 32 ++++++++++++++++++--
> >>> drivers/net/ethernet/intel/ice/ice_lib.c | 8 ++++-
> >>> drivers/net/ethernet/intel/ice/ice_main.c | 18 +++++------
> >>> 3 files changed, 45 insertions(+), 13 deletions(-)
> >>>
> >>> diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
> >>> index b82ff9556a4b..4f048268db72 100644
> >>> --- a/drivers/net/ethernet/intel/ice/ice_devlink.c
> >>> +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
> >>> @@ -1675,6 +1675,19 @@ ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
> >>> return 0;
> >>> }
> >>>
> >>> +static int ice_devlink_enable_rdma_validate(struct devlink *devlink, u32 id,
> >>> + union devlink_param_value val,
> >>> + struct netlink_ext_ack *extack)
> >>> +{
> >>> + struct ice_pf *pf = devlink_priv(devlink);
> >>> + bool new_state = val.vbool;
> >>> +
> >>> + if (new_state && !test_bit(ICE_FLAG_RDMA_ENA, pf->flags))
> >>> + return -EOPNOTSUPP;
> >>> +
> >>> + return 0;
> >>> +}
> >>> +
> >>> static const struct devlink_param ice_devlink_params[] = {
> >>> DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
> >>> ice_devlink_enable_roce_get,
> >>> @@ -1700,6 +1713,8 @@ static const struct devlink_param ice_devlink_params[] = {
> >>> ice_devlink_msix_min_pf_get,
> >>> ice_devlink_msix_min_pf_set,
> >>> ice_devlink_msix_min_pf_validate),
> >>> + DEVLINK_PARAM_GENERIC(ENABLE_RDMA, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
> >>> + NULL, NULL, ice_devlink_enable_rdma_validate),
> >>> };
> >>>
> >>> static void ice_devlink_free(void *devlink_ptr)
> >>> @@ -1776,9 +1791,22 @@ ice_devlink_set_switch_id(struct ice_pf *pf, struct netdev_phys_item_id *ppid)
> >>> int ice_devlink_register_params(struct ice_pf *pf)
> >>> {
> >>> struct devlink *devlink = priv_to_devlink(pf);
> >>> + union devlink_param_value value;
> >>> + int err;
> >>> +
> >>> + err = devlink_params_register(devlink, ice_devlink_params,
> >>
> >> Interesting, can't you just take the lock before this and call
> >> devl_params_register()?
> >>
> > I mess up a locking here and also in subfunction patchset. I will follow
> > you suggestion and take lock for whole init/cleanup. Thanks.
> >
> >> Moreover, could you please fix your init/cleanup paths for hold devlink
> >> instance lock the whole time?
> >>
> > You right, I will prepare patch for it.
>
> I think my patch implements your suggestion Jiri:
> https://lore.kernel.org/netdev/20240212211202.1051990-5-anthony.l.nguyen@intel.com/
>
Right, I based my patchset before your changes was applied to Tony's
tree. Thanks Wojtek.
So, in v2 there will be dev_version.
Thanks
Michal
> >
> >>
> >> pw-bot: cr
> >>
> >>
> >>> + ARRAY_SIZE(ice_devlink_params));
> >>> + if (err)
> >>> + return err;
> >>>
> >>> - return devlink_params_register(devlink, ice_devlink_params,
> >>> - ARRAY_SIZE(ice_devlink_params));
> >>> + devl_lock(devlink);
> >>> + value.vbool = test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
> >>> + devl_param_driverinit_value_set(devlink,
> >>> + DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
> >>> + value);
> >>> + devl_unlock(devlink);
> >>> +
> >>> + return 0;
> >>> }
> >>>
> >>> void ice_devlink_unregister_params(struct ice_pf *pf)
> >>> diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
> >>> index a30d2d2b51c1..4d5c3d699044 100644
> >>> --- a/drivers/net/ethernet/intel/ice/ice_lib.c
> >>> +++ b/drivers/net/ethernet/intel/ice/ice_lib.c
> >>> @@ -829,7 +829,13 @@ bool ice_is_safe_mode(struct ice_pf *pf)
> >>> */
> >>> bool ice_is_rdma_ena(struct ice_pf *pf)
> >>> {
> >>> - return test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
> >>> + union devlink_param_value value;
> >>> + int err;
> >>> +
> >>> + err = devl_param_driverinit_value_get(priv_to_devlink(pf),
> >>> + DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
> >>> + &value);
> >>> + return err ? false : value.vbool;
> >>> }
> >>>
> >>> /**
> >>> diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
> >>> index 824732f16112..4dd59d888ec7 100644
> >>> --- a/drivers/net/ethernet/intel/ice/ice_main.c
> >>> +++ b/drivers/net/ethernet/intel/ice/ice_main.c
> >>> @@ -5177,23 +5177,21 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
> >>> if (err)
> >>> goto err_init;
> >>>
> >>> + err = ice_init_devlink(pf);
> >>> + if (err)
> >>> + goto err_init_devlink;
> >>> +
> >>> devl_lock(priv_to_devlink(pf));
> >>> err = ice_load(pf);
> >>> devl_unlock(priv_to_devlink(pf));
> >>> if (err)
> >>> goto err_load;
> >>>
> >>> - err = ice_init_devlink(pf);
> >>> - if (err)
> >>> - goto err_init_devlink;
> >>> -
> >>> return 0;
> >>>
> >>> -err_init_devlink:
> >>> - devl_lock(priv_to_devlink(pf));
> >>> - ice_unload(pf);
> >>> - devl_unlock(priv_to_devlink(pf));
> >>> err_load:
> >>> + ice_deinit_devlink(pf);
> >>> +err_init_devlink:
> >>> ice_deinit(pf);
> >>> err_init:
> >>> pci_disable_device(pdev);
> >>> @@ -5290,12 +5288,12 @@ static void ice_remove(struct pci_dev *pdev)
> >>> if (!ice_is_safe_mode(pf))
> >>> ice_remove_arfs(pf);
> >>>
> >>> - ice_deinit_devlink(pf);
> >>> -
> >>> devl_lock(priv_to_devlink(pf));
> >>> ice_unload(pf);
> >>> devl_unlock(priv_to_devlink(pf));
> >>>
> >>> + ice_deinit_devlink(pf);
> >>> +
> >>> ice_deinit(pf);
> >>> ice_vsi_release_all(pf);
> >>>
> >>> --
> >>> 2.42.0
> >>>
> >>>
^ permalink raw reply [flat|nested] 17+ messages in thread* Re: [iwl-next v1 6/7] ice: enable_rdma devlink param
2024-02-13 10:19 ` Wojciech Drewek
2024-02-13 10:48 ` Michal Swiatkowski
@ 2024-02-13 11:35 ` Jiri Pirko
1 sibling, 0 replies; 17+ messages in thread
From: Jiri Pirko @ 2024-02-13 11:35 UTC (permalink / raw)
To: Wojciech Drewek
Cc: Michal Swiatkowski, intel-wired-lan, netdev, pawel.chmielewski,
sridhar.samudrala, jacob.e.keller, pio.raczynski, konrad.knitter,
marcin.szycik, nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel,
Jan Sokolowski
Tue, Feb 13, 2024 at 11:19:49AM CET, wojciech.drewek@intel.com wrote:
>
>
>On 13.02.2024 10:58, Michal Swiatkowski wrote:
>> On Tue, Feb 13, 2024 at 10:07:17AM +0100, Jiri Pirko wrote:
>>> Tue, Feb 13, 2024 at 08:35:08AM CET, michal.swiatkowski@linux.intel.com wrote:
>>>> Implement enable_rdma devlink parameter to allow user to turn RDMA
>>>> feature on and off.
>>>>
>>>> It is useful when there is no enough interrupts and user doesn't need
>>>> RDMA feature.
>>>>
>>>> Reviewed-by: Jan Sokolowski <jan.sokolowski@intel.com>
>>>> Reviewed-by: Przemek Kitszel <przemyslaw.kitszel@intel.com>
>>>> Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
>>>> ---
>>>> drivers/net/ethernet/intel/ice/ice_devlink.c | 32 ++++++++++++++++++--
>>>> drivers/net/ethernet/intel/ice/ice_lib.c | 8 ++++-
>>>> drivers/net/ethernet/intel/ice/ice_main.c | 18 +++++------
>>>> 3 files changed, 45 insertions(+), 13 deletions(-)
>>>>
>>>> diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
>>>> index b82ff9556a4b..4f048268db72 100644
>>>> --- a/drivers/net/ethernet/intel/ice/ice_devlink.c
>>>> +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
>>>> @@ -1675,6 +1675,19 @@ ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
>>>> return 0;
>>>> }
>>>>
>>>> +static int ice_devlink_enable_rdma_validate(struct devlink *devlink, u32 id,
>>>> + union devlink_param_value val,
>>>> + struct netlink_ext_ack *extack)
>>>> +{
>>>> + struct ice_pf *pf = devlink_priv(devlink);
>>>> + bool new_state = val.vbool;
>>>> +
>>>> + if (new_state && !test_bit(ICE_FLAG_RDMA_ENA, pf->flags))
>>>> + return -EOPNOTSUPP;
>>>> +
>>>> + return 0;
>>>> +}
>>>> +
>>>> static const struct devlink_param ice_devlink_params[] = {
>>>> DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
>>>> ice_devlink_enable_roce_get,
>>>> @@ -1700,6 +1713,8 @@ static const struct devlink_param ice_devlink_params[] = {
>>>> ice_devlink_msix_min_pf_get,
>>>> ice_devlink_msix_min_pf_set,
>>>> ice_devlink_msix_min_pf_validate),
>>>> + DEVLINK_PARAM_GENERIC(ENABLE_RDMA, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
>>>> + NULL, NULL, ice_devlink_enable_rdma_validate),
>>>> };
>>>>
>>>> static void ice_devlink_free(void *devlink_ptr)
>>>> @@ -1776,9 +1791,22 @@ ice_devlink_set_switch_id(struct ice_pf *pf, struct netdev_phys_item_id *ppid)
>>>> int ice_devlink_register_params(struct ice_pf *pf)
>>>> {
>>>> struct devlink *devlink = priv_to_devlink(pf);
>>>> + union devlink_param_value value;
>>>> + int err;
>>>> +
>>>> + err = devlink_params_register(devlink, ice_devlink_params,
>>>
>>> Interesting, can't you just take the lock before this and call
>>> devl_params_register()?
>>>
>> I mess up a locking here and also in subfunction patchset. I will follow
>> you suggestion and take lock for whole init/cleanup. Thanks.
>>
>>> Moreover, could you please fix your init/cleanup paths for hold devlink
>>> instance lock the whole time?
>>>
>> You right, I will prepare patch for it.
>
>I think my patch implements your suggestion Jiri:
>https://lore.kernel.org/netdev/20240212211202.1051990-5-anthony.l.nguyen@intel.com/
This patch is based on the one in your link apparently. I think it is
incomplete. Idk
>
>>
>>>
>>> pw-bot: cr
>>>
>>>
>>>> + ARRAY_SIZE(ice_devlink_params));
>>>> + if (err)
>>>> + return err;
>>>>
>>>> - return devlink_params_register(devlink, ice_devlink_params,
>>>> - ARRAY_SIZE(ice_devlink_params));
>>>> + devl_lock(devlink);
>>>> + value.vbool = test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
>>>> + devl_param_driverinit_value_set(devlink,
>>>> + DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
>>>> + value);
>>>> + devl_unlock(devlink);
>>>> +
>>>> + return 0;
>>>> }
>>>>
>>>> void ice_devlink_unregister_params(struct ice_pf *pf)
>>>> diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
>>>> index a30d2d2b51c1..4d5c3d699044 100644
>>>> --- a/drivers/net/ethernet/intel/ice/ice_lib.c
>>>> +++ b/drivers/net/ethernet/intel/ice/ice_lib.c
>>>> @@ -829,7 +829,13 @@ bool ice_is_safe_mode(struct ice_pf *pf)
>>>> */
>>>> bool ice_is_rdma_ena(struct ice_pf *pf)
>>>> {
>>>> - return test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
>>>> + union devlink_param_value value;
>>>> + int err;
>>>> +
>>>> + err = devl_param_driverinit_value_get(priv_to_devlink(pf),
>>>> + DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
>>>> + &value);
>>>> + return err ? false : value.vbool;
>>>> }
>>>>
>>>> /**
>>>> diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
>>>> index 824732f16112..4dd59d888ec7 100644
>>>> --- a/drivers/net/ethernet/intel/ice/ice_main.c
>>>> +++ b/drivers/net/ethernet/intel/ice/ice_main.c
>>>> @@ -5177,23 +5177,21 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
>>>> if (err)
>>>> goto err_init;
>>>>
>>>> + err = ice_init_devlink(pf);
>>>> + if (err)
>>>> + goto err_init_devlink;
>>>> +
>>>> devl_lock(priv_to_devlink(pf));
>>>> err = ice_load(pf);
>>>> devl_unlock(priv_to_devlink(pf));
>>>> if (err)
>>>> goto err_load;
>>>>
>>>> - err = ice_init_devlink(pf);
>>>> - if (err)
>>>> - goto err_init_devlink;
>>>> -
>>>> return 0;
>>>>
>>>> -err_init_devlink:
>>>> - devl_lock(priv_to_devlink(pf));
>>>> - ice_unload(pf);
>>>> - devl_unlock(priv_to_devlink(pf));
>>>> err_load:
>>>> + ice_deinit_devlink(pf);
>>>> +err_init_devlink:
>>>> ice_deinit(pf);
>>>> err_init:
>>>> pci_disable_device(pdev);
>>>> @@ -5290,12 +5288,12 @@ static void ice_remove(struct pci_dev *pdev)
>>>> if (!ice_is_safe_mode(pf))
>>>> ice_remove_arfs(pf);
>>>>
>>>> - ice_deinit_devlink(pf);
>>>> -
>>>> devl_lock(priv_to_devlink(pf));
>>>> ice_unload(pf);
>>>> devl_unlock(priv_to_devlink(pf));
>>>>
>>>> + ice_deinit_devlink(pf);
>>>> +
>>>> ice_deinit(pf);
>>>> ice_vsi_release_all(pf);
>>>>
>>>> --
>>>> 2.42.0
>>>>
>>>>
^ permalink raw reply [flat|nested] 17+ messages in thread
* [iwl-next v1 7/7] ice: simplify VF MSI-X managing
2024-02-13 7:35 [iwl-next v1 0/7] ice: managing MSI-X in driver Michal Swiatkowski
` (5 preceding siblings ...)
2024-02-13 7:35 ` [iwl-next v1 6/7] ice: enable_rdma devlink param Michal Swiatkowski
@ 2024-02-13 7:35 ` Michal Swiatkowski
6 siblings, 0 replies; 17+ messages in thread
From: Michal Swiatkowski @ 2024-02-13 7:35 UTC (permalink / raw)
To: intel-wired-lan
Cc: netdev, pawel.chmielewski, sridhar.samudrala, jacob.e.keller,
pio.raczynski, konrad.knitter, marcin.szycik, wojciech.drewek,
nex.sw.ncis.nat.hpm.dev, przemyslaw.kitszel, Michal Swiatkowski
After implementing pf->msix.max field, base vector for other use cases
(like VFs) can be fixed. This simplify code when changing MSI-X amount
on particular VF, because there is no need to move a base vector.
A fixed base vector allows to reserve vectors from the beginning
instead of from the end, which is also simpler in code.
Store total and rest value in the same struct as max and min for PF.
Move tracking vectors from ice_sriov.c to ice_irq.c as it can be also
use for other none PF use cases (SIOV).
Signed-off-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com>
---
drivers/net/ethernet/intel/ice/ice.h | 10 +-
drivers/net/ethernet/intel/ice/ice_irq.c | 79 +++++++----
drivers/net/ethernet/intel/ice/ice_irq.h | 13 +-
drivers/net/ethernet/intel/ice/ice_sriov.c | 153 ++-------------------
4 files changed, 80 insertions(+), 175 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 1f4d31549528..89afab4fbe50 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -538,6 +538,8 @@ struct ice_pf_msix {
u16 cur;
u16 min;
u16 max;
+ u16 total;
+ u16 rest;
};
struct ice_pf {
@@ -553,13 +555,7 @@ struct ice_pf {
/* OS reserved IRQ details */
struct msix_entry *msix_entries;
struct ice_irq_tracker irq_tracker;
- /* First MSIX vector used by SR-IOV VFs. Calculated by subtracting the
- * number of MSIX vectors needed for all SR-IOV VFs from the number of
- * MSIX vectors allowed on this PF.
- */
- u16 sriov_base_vector;
- unsigned long *sriov_irq_bm; /* bitmap to track irq usage */
- u16 sriov_irq_size; /* size of the irq_bm bitmap */
+ struct ice_virt_irq_tracker virt_irq_tracker;
u16 ctrl_vsi_idx; /* control VSI index in pf->vsi array */
diff --git a/drivers/net/ethernet/intel/ice/ice_irq.c b/drivers/net/ethernet/intel/ice/ice_irq.c
index f95f6e5987b7..41ca3829adef 100644
--- a/drivers/net/ethernet/intel/ice/ice_irq.c
+++ b/drivers/net/ethernet/intel/ice/ice_irq.c
@@ -20,6 +20,19 @@ ice_init_irq_tracker(struct ice_pf *pf, unsigned int max_vectors,
xa_init_flags(&pf->irq_tracker.entries, XA_FLAGS_ALLOC);
}
+static int
+ice_init_virt_irq_tracker(struct ice_pf *pf, u16 base, u16 num_entries)
+{
+ pf->virt_irq_tracker.bm = bitmap_zalloc(num_entries, GFP_KERNEL);
+ if (!pf->virt_irq_tracker.bm)
+ return -ENOMEM;
+
+ pf->virt_irq_tracker.num_entries = num_entries;
+ pf->virt_irq_tracker.base = base;
+
+ return 0;
+}
+
/**
* ice_deinit_irq_tracker - free xarray tracker
* @pf: board private structure
@@ -29,6 +42,11 @@ static void ice_deinit_irq_tracker(struct ice_pf *pf)
xa_destroy(&pf->irq_tracker.entries);
}
+static void ice_deinit_virt_irq_tracker(struct ice_pf *pf)
+{
+ bitmap_free(pf->virt_irq_tracker.bm);
+}
+
/**
* ice_free_irq_res - free a block of resources
* @pf: board private structure
@@ -93,6 +111,7 @@ void ice_clear_interrupt_scheme(struct ice_pf *pf)
{
pci_free_irq_vectors(pf->pdev);
ice_deinit_irq_tracker(pf);
+ ice_deinit_virt_irq_tracker(pf);
}
/**
@@ -105,10 +124,13 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
int vectors;
/* load default PF MSI-X range */
- if (!pf->msix.min)
+ if (!pf->msix.total) {
pf->msix.min = ICE_MIN_MSIX;
- if (!pf->msix.max)
pf->msix.max = total_vectors / 2;
+ }
+
+ pf->msix.total = total_vectors;
+ pf->msix.rest = total_vectors - pf->msix.max;
if (pci_msix_can_alloc_dyn(pf->pdev))
vectors = pf->msix.min;
@@ -122,7 +144,7 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
ice_init_irq_tracker(pf, pf->msix.max, vectors);
- return 0;
+ return ice_init_virt_irq_tracker(pf, pf->msix.max, pf->msix.rest);
}
/**
@@ -149,7 +171,6 @@ int ice_init_interrupt_scheme(struct ice_pf *pf)
*/
struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_allowed)
{
- int sriov_base_vector = pf->sriov_base_vector;
struct msi_map map = { .index = -ENOENT };
struct device *dev = ice_pf_to_dev(pf);
struct ice_irq_entry *entry;
@@ -158,10 +179,6 @@ struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_allowed)
if (!entry)
return map;
- /* fail if we're about to violate SRIOV vectors space */
- if (sriov_base_vector && entry->index >= sriov_base_vector)
- goto exit_free_res;
-
if (pci_msix_can_alloc_dyn(pf->pdev) && entry->dynamic) {
map = pci_msix_alloc_irq_at(pf->pdev, entry->index, NULL);
if (map.index < 0)
@@ -209,26 +226,40 @@ void ice_free_irq(struct ice_pf *pf, struct msi_map map)
}
/**
- * ice_get_max_used_msix_vector - Get the max used interrupt vector
- * @pf: board private structure
+ * ice_virt_get_irqs - get irqs for SR-IOV usacase
+ * @pf: pointer to PF structure
+ * @needed: number of irqs to get
*
- * Return index of maximum used interrupt vectors with respect to the
- * beginning of the MSIX table. Take into account that some interrupts
- * may have been dynamically allocated after MSIX was initially enabled.
+ * This returns the first MSI-X vector index in PF space that is used by this
+ * VF. This index is used when accessing PF relative registers such as
+ * GLINT_VECT2FUNC and GLINT_DYN_CTL.
+ * This will always be the OICR index in the AVF driver so any functionality
+ * using vf->first_vector_idx for queue configuration_id: id of VF which will
+ * use this irqs
*/
-int ice_get_max_used_msix_vector(struct ice_pf *pf)
+int ice_virt_get_irqs(struct ice_pf *pf, u16 needed)
{
- unsigned long start, index, max_idx;
- void *entry;
+ int res = bitmap_find_next_zero_area(pf->virt_irq_tracker.bm,
+ pf->virt_irq_tracker.num_entries,
+ 0, needed, 0);
- /* Treat all preallocated interrupts as used */
- start = pf->irq_tracker.num_static;
- max_idx = start - 1;
+ if (res >= pf->virt_irq_tracker.num_entries)
+ return -ENOENT;
- xa_for_each_start(&pf->irq_tracker.entries, index, entry, start) {
- if (index > max_idx)
- max_idx = index;
- }
+ bitmap_set(pf->virt_irq_tracker.bm, res, needed);
- return max_idx;
+ /* conversion from number in bitmap to global irq index */
+ return res + pf->virt_irq_tracker.base;
+}
+
+/**
+ * ice_virt_free_irqs - free irqs used by the VF
+ * @pf: pointer to PF structure
+ * @index: first index to be free
+ * @irqs: number of irqs to free
+ */
+void ice_virt_free_irqs(struct ice_pf *pf, u16 index, u16 irqs)
+{
+ bitmap_clear(pf->virt_irq_tracker.bm, index - pf->virt_irq_tracker.base,
+ irqs);
}
diff --git a/drivers/net/ethernet/intel/ice/ice_irq.h b/drivers/net/ethernet/intel/ice/ice_irq.h
index f35efc08575e..d5e0fdd9b535 100644
--- a/drivers/net/ethernet/intel/ice/ice_irq.h
+++ b/drivers/net/ethernet/intel/ice/ice_irq.h
@@ -15,11 +15,22 @@ struct ice_irq_tracker {
u16 num_static; /* preallocated entries */
};
+struct ice_virt_irq_tracker {
+ unsigned long *bm; /* bitmap to track irq usage */
+ u16 num_entries;
+ /* First MSIX vector used by SR-IOV VFs. Calculated by subtracting the
+ * number of MSIX vectors needed for all SR-IOV VFs from the number of
+ * MSIX vectors allowed on this PF.
+ */
+ u16 base;
+};
+
int ice_init_interrupt_scheme(struct ice_pf *pf);
void ice_clear_interrupt_scheme(struct ice_pf *pf);
struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_only);
void ice_free_irq(struct ice_pf *pf, struct msi_map map);
-int ice_get_max_used_msix_vector(struct ice_pf *pf);
+int ice_virt_get_irqs(struct ice_pf *pf, u16 needed);
+void ice_virt_free_irqs(struct ice_pf *pf, u16 index, u16 irqs);
#endif
diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c
index 706b5ee8ec89..48e78379518c 100644
--- a/drivers/net/ethernet/intel/ice/ice_sriov.c
+++ b/drivers/net/ethernet/intel/ice/ice_sriov.c
@@ -122,27 +122,6 @@ static void ice_dis_vf_mappings(struct ice_vf *vf)
dev_err(dev, "Scattered mode for VF Rx queues is not yet implemented\n");
}
-/**
- * ice_sriov_free_msix_res - Reset/free any used MSIX resources
- * @pf: pointer to the PF structure
- *
- * Since no MSIX entries are taken from the pf->irq_tracker then just clear
- * the pf->sriov_base_vector.
- *
- * Returns 0 on success, and -EINVAL on error.
- */
-static int ice_sriov_free_msix_res(struct ice_pf *pf)
-{
- if (!pf)
- return -EINVAL;
-
- bitmap_free(pf->sriov_irq_bm);
- pf->sriov_irq_size = 0;
- pf->sriov_base_vector = 0;
-
- return 0;
-}
-
/**
* ice_free_vfs - Free all VFs
* @pf: pointer to the PF structure
@@ -177,6 +156,7 @@ void ice_free_vfs(struct ice_pf *pf)
ice_eswitch_detach(pf, vf);
ice_dis_vf_qs(vf);
+ ice_virt_free_irqs(pf, vf->first_vector_idx, vf->num_msix);
if (test_bit(ICE_VF_STATE_INIT, vf->vf_states)) {
/* disable VF qp mappings and set VF disable state */
@@ -199,9 +179,6 @@ void ice_free_vfs(struct ice_pf *pf)
mutex_unlock(&vf->cfg_lock);
}
- if (ice_sriov_free_msix_res(pf))
- dev_err(dev, "Failed to free MSIX resources used by SR-IOV\n");
-
vfs->num_qps_per = 0;
ice_free_vf_entries(pf);
@@ -370,40 +347,6 @@ int ice_calc_vf_reg_idx(struct ice_vf *vf, struct ice_q_vector *q_vector)
return vf->first_vector_idx + q_vector->v_idx + 1;
}
-/**
- * ice_sriov_set_msix_res - Set any used MSIX resources
- * @pf: pointer to PF structure
- * @num_msix_needed: number of MSIX vectors needed for all SR-IOV VFs
- *
- * This function allows SR-IOV resources to be taken from the end of the PF's
- * allowed HW MSIX vectors so that the irq_tracker will not be affected. We
- * just set the pf->sriov_base_vector and return success.
- *
- * If there are not enough resources available, return an error. This should
- * always be caught by ice_set_per_vf_res().
- *
- * Return 0 on success, and -EINVAL when there are not enough MSIX vectors
- * in the PF's space available for SR-IOV.
- */
-static int ice_sriov_set_msix_res(struct ice_pf *pf, u16 num_msix_needed)
-{
- u16 total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
- int vectors_used = ice_get_max_used_msix_vector(pf);
- int sriov_base_vector;
-
- sriov_base_vector = total_vectors - num_msix_needed;
-
- /* make sure we only grab irq_tracker entries from the list end and
- * that we have enough available MSIX vectors
- */
- if (sriov_base_vector < vectors_used)
- return -EINVAL;
-
- pf->sriov_base_vector = sriov_base_vector;
-
- return 0;
-}
-
/**
* ice_set_per_vf_res - check if vectors and queues are available
* @pf: pointer to the PF structure
@@ -428,11 +371,9 @@ static int ice_sriov_set_msix_res(struct ice_pf *pf, u16 num_msix_needed)
*/
static int ice_set_per_vf_res(struct ice_pf *pf, u16 num_vfs)
{
- int vectors_used = ice_get_max_used_msix_vector(pf);
u16 num_msix_per_vf, num_txq, num_rxq, avail_qs;
int msix_avail_per_vf, msix_avail_for_sriov;
struct device *dev = ice_pf_to_dev(pf);
- int err;
lockdep_assert_held(&pf->vfs.table_lock);
@@ -440,8 +381,7 @@ static int ice_set_per_vf_res(struct ice_pf *pf, u16 num_vfs)
return -EINVAL;
/* determine MSI-X resources per VF */
- msix_avail_for_sriov = pf->hw.func_caps.common_cap.num_msix_vectors -
- vectors_used;
+ msix_avail_for_sriov = pf->virt_irq_tracker.num_entries;
msix_avail_per_vf = msix_avail_for_sriov / num_vfs;
if (msix_avail_per_vf >= ICE_NUM_VF_MSIX_MED) {
num_msix_per_vf = ICE_NUM_VF_MSIX_MED;
@@ -480,13 +420,6 @@ static int ice_set_per_vf_res(struct ice_pf *pf, u16 num_vfs)
return -ENOSPC;
}
- err = ice_sriov_set_msix_res(pf, num_msix_per_vf * num_vfs);
- if (err) {
- dev_err(dev, "Unable to set MSI-X resources for %d VFs, err %d\n",
- num_vfs, err);
- return err;
- }
-
/* only allow equal Tx/Rx queue count (i.e. queue pairs) */
pf->vfs.num_qps_per = min_t(int, num_txq, num_rxq);
pf->vfs.num_msix_per = num_msix_per_vf;
@@ -496,52 +429,6 @@ static int ice_set_per_vf_res(struct ice_pf *pf, u16 num_vfs)
return 0;
}
-/**
- * ice_sriov_get_irqs - get irqs for SR-IOV usacase
- * @pf: pointer to PF structure
- * @needed: number of irqs to get
- *
- * This returns the first MSI-X vector index in PF space that is used by this
- * VF. This index is used when accessing PF relative registers such as
- * GLINT_VECT2FUNC and GLINT_DYN_CTL.
- * This will always be the OICR index in the AVF driver so any functionality
- * using vf->first_vector_idx for queue configuration_id: id of VF which will
- * use this irqs
- *
- * Only SRIOV specific vectors are tracked in sriov_irq_bm. SRIOV vectors are
- * allocated from the end of global irq index. First bit in sriov_irq_bm means
- * last irq index etc. It simplifies extension of SRIOV vectors.
- * They will be always located from sriov_base_vector to the last irq
- * index. While increasing/decreasing sriov_base_vector can be moved.
- */
-static int ice_sriov_get_irqs(struct ice_pf *pf, u16 needed)
-{
- int res = bitmap_find_next_zero_area(pf->sriov_irq_bm,
- pf->sriov_irq_size, 0, needed, 0);
- /* conversion from number in bitmap to global irq index */
- int index = pf->sriov_irq_size - res - needed;
-
- if (res >= pf->sriov_irq_size || index < pf->sriov_base_vector)
- return -ENOENT;
-
- bitmap_set(pf->sriov_irq_bm, res, needed);
- return index;
-}
-
-/**
- * ice_sriov_free_irqs - free irqs used by the VF
- * @pf: pointer to PF structure
- * @vf: pointer to VF structure
- */
-static void ice_sriov_free_irqs(struct ice_pf *pf, struct ice_vf *vf)
-{
- /* Move back from first vector index to first index in bitmap */
- int bm_i = pf->sriov_irq_size - vf->first_vector_idx - vf->num_msix;
-
- bitmap_clear(pf->sriov_irq_bm, bm_i, vf->num_msix);
- vf->first_vector_idx = 0;
-}
-
/**
* ice_init_vf_vsi_res - initialize/setup VF VSI resources
* @vf: VF to initialize/setup the VSI for
@@ -555,7 +442,7 @@ static int ice_init_vf_vsi_res(struct ice_vf *vf)
struct ice_vsi *vsi;
int err;
- vf->first_vector_idx = ice_sriov_get_irqs(pf, vf->num_msix);
+ vf->first_vector_idx = ice_virt_get_irqs(pf, vf->num_msix);
if (vf->first_vector_idx < 0)
return -ENOMEM;
@@ -860,16 +747,10 @@ static int ice_create_vf_entries(struct ice_pf *pf, u16 num_vfs)
*/
static int ice_ena_vfs(struct ice_pf *pf, u16 num_vfs)
{
- int total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
struct device *dev = ice_pf_to_dev(pf);
struct ice_hw *hw = &pf->hw;
int ret;
- pf->sriov_irq_bm = bitmap_zalloc(total_vectors, GFP_KERNEL);
- if (!pf->sriov_irq_bm)
- return -ENOMEM;
- pf->sriov_irq_size = total_vectors;
-
/* Disable global interrupt 0 so we don't try to handle the VFLR. */
wr32(hw, GLINT_DYN_CTL(pf->oicr_irq.index),
ICE_ITR_NONE << GLINT_DYN_CTL_ITR_INDX_S);
@@ -922,7 +803,6 @@ static int ice_ena_vfs(struct ice_pf *pf, u16 num_vfs)
/* rearm interrupts here */
ice_irq_dynamic_ena(hw, NULL, NULL);
clear_bit(ICE_OICR_INTR_DIS, pf->state);
- bitmap_free(pf->sriov_irq_bm);
return ret;
}
@@ -996,16 +876,7 @@ u32 ice_sriov_get_vf_total_msix(struct pci_dev *pdev)
{
struct ice_pf *pf = pci_get_drvdata(pdev);
- return pf->sriov_irq_size - ice_get_max_used_msix_vector(pf);
-}
-
-static int ice_sriov_move_base_vector(struct ice_pf *pf, int move)
-{
- if (pf->sriov_base_vector - move < ice_get_max_used_msix_vector(pf))
- return -ENOMEM;
-
- pf->sriov_base_vector -= move;
- return 0;
+ return pf->virt_irq_tracker.num_entries;
}
static void ice_sriov_remap_vectors(struct ice_pf *pf, u16 restricted_id)
@@ -1024,7 +895,8 @@ static void ice_sriov_remap_vectors(struct ice_pf *pf, u16 restricted_id)
continue;
ice_dis_vf_mappings(tmp_vf);
- ice_sriov_free_irqs(pf, tmp_vf);
+ ice_virt_free_irqs(pf, tmp_vf->first_vector_idx,
+ tmp_vf->num_msix);
vf_ids[to_remap] = tmp_vf->vf_id;
to_remap += 1;
@@ -1036,7 +908,7 @@ static void ice_sriov_remap_vectors(struct ice_pf *pf, u16 restricted_id)
continue;
tmp_vf->first_vector_idx =
- ice_sriov_get_irqs(pf, tmp_vf->num_msix);
+ ice_virt_get_irqs(pf, tmp_vf->num_msix);
/* there is no need to rebuild VSI as we are only changing the
* vector indexes not amount of MSI-X or queues
*/
@@ -1102,20 +974,15 @@ int ice_sriov_set_msix_vec_count(struct pci_dev *vf_dev, int msix_vec_count)
prev_msix = vf->num_msix;
prev_queues = vf->num_vf_qs;
- if (ice_sriov_move_base_vector(pf, msix_vec_count - prev_msix)) {
- ice_put_vf(vf);
- return -ENOSPC;
- }
-
ice_dis_vf_mappings(vf);
- ice_sriov_free_irqs(pf, vf);
+ ice_virt_free_irqs(pf, vf->first_vector_idx, vf->num_msix);
/* Remap all VFs beside the one is now configured */
ice_sriov_remap_vectors(pf, vf->vf_id);
vf->num_msix = msix_vec_count;
vf->num_vf_qs = queues;
- vf->first_vector_idx = ice_sriov_get_irqs(pf, vf->num_msix);
+ vf->first_vector_idx = ice_virt_get_irqs(pf, vf->num_msix);
if (vf->first_vector_idx < 0)
goto unroll;
@@ -1141,7 +1008,7 @@ int ice_sriov_set_msix_vec_count(struct pci_dev *vf_dev, int msix_vec_count)
vf->num_msix = prev_msix;
vf->num_vf_qs = prev_queues;
- vf->first_vector_idx = ice_sriov_get_irqs(pf, vf->num_msix);
+ vf->first_vector_idx = ice_virt_get_irqs(pf, vf->num_msix);
if (vf->first_vector_idx < 0)
return -EINVAL;
--
2.42.0
^ permalink raw reply related [flat|nested] 17+ messages in thread