* [net-next 07/15] ice: Fix kernel hang with DCB reset in CEE mode
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem; +Cc: Usha Ketineni, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Usha Ketineni <usha.k.ketineni@intel.com>
This patch fixes the set local MIB AQ call failures in the DCB rebuild path
by setting the defaults for the ETS recommended DCB configuration. Also,
willing bits for the DCB configuration needs to be set correctly. Resets
works fine in IEEE mode as the ETS recommended DCB configuration is
populated but not in CEE mode.
Without this patch, PFR causes the kernel hang in CEE mode.
Signed-off-by: Usha Ketineni <usha.k.ketineni@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice_dcb_lib.c | 149 +++++++++++--------
1 file changed, 88 insertions(+), 61 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
index f80628a13f2a..d285aba3fea7 100644
--- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
@@ -208,16 +208,87 @@ int ice_pf_dcb_cfg(struct ice_pf *pf, struct ice_dcbx_cfg *new_cfg, bool locked)
return ret;
}
+/**
+ * ice_cfg_etsrec_defaults - Set default ETS recommended DCB config
+ * @pi: port information structure
+ */
+static void ice_cfg_etsrec_defaults(struct ice_port_info *pi)
+{
+ struct ice_dcbx_cfg *dcbcfg = &pi->local_dcbx_cfg;
+ u8 i;
+
+ /* Ensure ETS recommended DCB configuration is not already set */
+ if (dcbcfg->etsrec.maxtcs)
+ return;
+
+ /* In CEE mode, set the default to 1 TC */
+ dcbcfg->etsrec.maxtcs = 1;
+ for (i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
+ dcbcfg->etsrec.tcbwtable[i] = i ? 0 : 100;
+ dcbcfg->etsrec.tsatable[i] = i ? ICE_IEEE_TSA_STRICT :
+ ICE_IEEE_TSA_ETS;
+ }
+}
+
+/**
+ * ice_dcb_need_recfg - Check if DCB needs reconfig
+ * @pf: board private structure
+ * @old_cfg: current DCB config
+ * @new_cfg: new DCB config
+ */
+static bool
+ice_dcb_need_recfg(struct ice_pf *pf, struct ice_dcbx_cfg *old_cfg,
+ struct ice_dcbx_cfg *new_cfg)
+{
+ bool need_reconfig = false;
+
+ /* Check if ETS configuration has changed */
+ if (memcmp(&new_cfg->etscfg, &old_cfg->etscfg,
+ sizeof(new_cfg->etscfg))) {
+ /* If Priority Table has changed reconfig is needed */
+ if (memcmp(&new_cfg->etscfg.prio_table,
+ &old_cfg->etscfg.prio_table,
+ sizeof(new_cfg->etscfg.prio_table))) {
+ need_reconfig = true;
+ dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
+ }
+
+ if (memcmp(&new_cfg->etscfg.tcbwtable,
+ &old_cfg->etscfg.tcbwtable,
+ sizeof(new_cfg->etscfg.tcbwtable)))
+ dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
+
+ if (memcmp(&new_cfg->etscfg.tsatable,
+ &old_cfg->etscfg.tsatable,
+ sizeof(new_cfg->etscfg.tsatable)))
+ dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
+ }
+
+ /* Check if PFC configuration has changed */
+ if (memcmp(&new_cfg->pfc, &old_cfg->pfc, sizeof(new_cfg->pfc))) {
+ need_reconfig = true;
+ dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
+ }
+
+ /* Check if APP Table has changed */
+ if (memcmp(&new_cfg->app, &old_cfg->app, sizeof(new_cfg->app))) {
+ need_reconfig = true;
+ dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
+ }
+
+ dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
+ return need_reconfig;
+}
+
/**
* ice_dcb_rebuild - rebuild DCB post reset
* @pf: physical function instance
*/
void ice_dcb_rebuild(struct ice_pf *pf)
{
+ struct ice_dcbx_cfg *local_dcbx_cfg, *desired_dcbx_cfg, *prev_cfg;
struct ice_aqc_port_ets_elem buf = { 0 };
- struct ice_dcbx_cfg *prev_cfg;
enum ice_status ret;
- u8 willing;
ret = ice_query_port_ets(pf->hw.port_info, &buf, sizeof(buf), NULL);
if (ret) {
@@ -229,9 +300,15 @@ void ice_dcb_rebuild(struct ice_pf *pf)
if (!test_bit(ICE_FLAG_DCB_ENA, pf->flags))
return;
+ local_dcbx_cfg = &pf->hw.port_info->local_dcbx_cfg;
+ desired_dcbx_cfg = &pf->hw.port_info->desired_dcbx_cfg;
+
/* Save current willing state and force FW to unwilling */
- willing = pf->hw.port_info->local_dcbx_cfg.etscfg.willing;
- pf->hw.port_info->local_dcbx_cfg.etscfg.willing = 0x0;
+ local_dcbx_cfg->etscfg.willing = 0x0;
+ local_dcbx_cfg->pfc.willing = 0x0;
+ local_dcbx_cfg->app_mode = ICE_DCBX_APPS_NON_WILLING;
+
+ ice_cfg_etsrec_defaults(pf->hw.port_info);
ret = ice_set_dcb_cfg(pf->hw.port_info);
if (ret) {
dev_err(&pf->pdev->dev, "Failed to set DCB to unwilling\n");
@@ -239,8 +316,7 @@ void ice_dcb_rebuild(struct ice_pf *pf)
}
/* Retrieve DCB config and ensure same as current in SW */
- prev_cfg = devm_kmemdup(&pf->pdev->dev,
- &pf->hw.port_info->local_dcbx_cfg,
+ prev_cfg = devm_kmemdup(&pf->pdev->dev, local_dcbx_cfg,
sizeof(*prev_cfg), GFP_KERNEL);
if (!prev_cfg) {
dev_err(&pf->pdev->dev, "Failed to alloc space for DCB cfg\n");
@@ -248,22 +324,22 @@ void ice_dcb_rebuild(struct ice_pf *pf)
}
ice_init_dcb(&pf->hw);
- if (memcmp(prev_cfg, &pf->hw.port_info->local_dcbx_cfg,
- sizeof(*prev_cfg))) {
+ if (ice_dcb_need_recfg(pf, prev_cfg, local_dcbx_cfg)) {
/* difference in cfg detected - disable DCB till next MIB */
dev_err(&pf->pdev->dev, "Set local MIB not accurate\n");
- devm_kfree(&pf->pdev->dev, prev_cfg);
goto dcb_error;
}
/* fetched config congruent to previous configuration */
devm_kfree(&pf->pdev->dev, prev_cfg);
- /* Configuration replayed - reset willing state to previous */
- pf->hw.port_info->local_dcbx_cfg.etscfg.willing = willing;
+ /* Set the local desired config */
+ memset(&pf->hw.port_info->local_dcbx_cfg, 0, sizeof(*local_dcbx_cfg));
+ memcpy(local_dcbx_cfg, desired_dcbx_cfg, sizeof(*local_dcbx_cfg));
+ ice_cfg_etsrec_defaults(pf->hw.port_info);
ret = ice_set_dcb_cfg(pf->hw.port_info);
if (ret) {
- dev_err(&pf->pdev->dev, "Fail restoring prev willing state\n");
+ dev_err(&pf->pdev->dev, "Failed to set desired config\n");
goto dcb_error;
}
dev_info(&pf->pdev->dev, "DCB restored after reset\n");
@@ -506,55 +582,6 @@ ice_tx_prepare_vlan_flags_dcb(struct ice_ring *tx_ring,
return 0;
}
-/**
- * ice_dcb_need_recfg - Check if DCB needs reconfig
- * @pf: board private structure
- * @old_cfg: current DCB config
- * @new_cfg: new DCB config
- */
-static bool ice_dcb_need_recfg(struct ice_pf *pf, struct ice_dcbx_cfg *old_cfg,
- struct ice_dcbx_cfg *new_cfg)
-{
- bool need_reconfig = false;
-
- /* Check if ETS configuration has changed */
- if (memcmp(&new_cfg->etscfg, &old_cfg->etscfg,
- sizeof(new_cfg->etscfg))) {
- /* If Priority Table has changed reconfig is needed */
- if (memcmp(&new_cfg->etscfg.prio_table,
- &old_cfg->etscfg.prio_table,
- sizeof(new_cfg->etscfg.prio_table))) {
- need_reconfig = true;
- dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
- }
-
- if (memcmp(&new_cfg->etscfg.tcbwtable,
- &old_cfg->etscfg.tcbwtable,
- sizeof(new_cfg->etscfg.tcbwtable)))
- dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
-
- if (memcmp(&new_cfg->etscfg.tsatable,
- &old_cfg->etscfg.tsatable,
- sizeof(new_cfg->etscfg.tsatable)))
- dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
- }
-
- /* Check if PFC configuration has changed */
- if (memcmp(&new_cfg->pfc, &old_cfg->pfc, sizeof(new_cfg->pfc))) {
- need_reconfig = true;
- dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
- }
-
- /* Check if APP Table has changed */
- if (memcmp(&new_cfg->app, &old_cfg->app, sizeof(new_cfg->app))) {
- need_reconfig = true;
- dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
- }
-
- dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
- return need_reconfig;
-}
-
/**
* ice_dcb_process_lldp_set_mib_change - Process MIB change
* @pf: ptr to ice_pf
--
2.21.0
^ permalink raw reply related
* [net-next 10/15] ice: update GLINT_DYN_CTL and GLINT_VECT2FUNC register access
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem
Cc: Paul Greenwalt, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Paul Greenwalt <paul.greenwalt@intel.com>
Register access for GLINT_DYN_CTL and GLINT_VECT2FUNC should be within
the PF space and not the absolute device space.
Signed-off-by: Paul Greenwalt <paul.greenwalt@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
.../net/ethernet/intel/ice/ice_virtchnl_pf.c | 24 +++++++++++--------
.../net/ethernet/intel/ice/ice_virtchnl_pf.h | 3 ++-
2 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
index 93b835a24bb1..54b0e88af4ca 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
@@ -474,19 +474,20 @@ ice_vf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, u16 vf_id)
}
/**
- * ice_calc_vf_first_vector_idx - Calculate absolute MSIX vector index in HW
+ * ice_calc_vf_first_vector_idx - Calculate MSIX vector index in the PF space
* @pf: pointer to PF structure
* @vf: pointer to VF that the first MSIX vector index is being calculated for
*
- * This returns the first MSIX vector index in HW that is used by this VF and
- * this will always be the OICR index in the AVF driver so any functionality
+ * This returns the first MSIX vector index in PF space that is used by this VF.
+ * This index is used when accessing PF relative registers such as
+ * GLINT_VECT2FUNC and GLINT_DYN_CTL.
+ * This will always be the OICR index in the AVF driver so any functionality
* using vf->first_vector_idx for queue configuration will have to increment by
* 1 to avoid meddling with the OICR index.
*/
static int ice_calc_vf_first_vector_idx(struct ice_pf *pf, struct ice_vf *vf)
{
- return pf->hw.func_caps.common_cap.msix_vector_first_id +
- pf->sriov_base_vector + vf->vf_id * pf->num_vf_msix;
+ return pf->sriov_base_vector + vf->vf_id * pf->num_vf_msix;
}
/**
@@ -597,27 +598,30 @@ static int ice_alloc_vf_res(struct ice_vf *vf)
*/
static void ice_ena_vf_mappings(struct ice_vf *vf)
{
+ int abs_vf_id, abs_first, abs_last;
struct ice_pf *pf = vf->pf;
struct ice_vsi *vsi;
int first, last, v;
struct ice_hw *hw;
- int abs_vf_id;
u32 reg;
hw = &pf->hw;
vsi = pf->vsi[vf->lan_vsi_idx];
first = vf->first_vector_idx;
last = (first + pf->num_vf_msix) - 1;
+ abs_first = first + pf->hw.func_caps.common_cap.msix_vector_first_id;
+ abs_last = (abs_first + pf->num_vf_msix) - 1;
abs_vf_id = vf->vf_id + hw->func_caps.vf_base_id;
/* VF Vector allocation */
- reg = (((first << VPINT_ALLOC_FIRST_S) & VPINT_ALLOC_FIRST_M) |
- ((last << VPINT_ALLOC_LAST_S) & VPINT_ALLOC_LAST_M) |
+ reg = (((abs_first << VPINT_ALLOC_FIRST_S) & VPINT_ALLOC_FIRST_M) |
+ ((abs_last << VPINT_ALLOC_LAST_S) & VPINT_ALLOC_LAST_M) |
VPINT_ALLOC_VALID_M);
wr32(hw, VPINT_ALLOC(vf->vf_id), reg);
- reg = (((first << VPINT_ALLOC_PCI_FIRST_S) & VPINT_ALLOC_PCI_FIRST_M) |
- ((last << VPINT_ALLOC_PCI_LAST_S) & VPINT_ALLOC_PCI_LAST_M) |
+ reg = (((abs_first << VPINT_ALLOC_PCI_FIRST_S)
+ & VPINT_ALLOC_PCI_FIRST_M) |
+ ((abs_last << VPINT_ALLOC_PCI_LAST_S) & VPINT_ALLOC_PCI_LAST_M) |
VPINT_ALLOC_PCI_VALID_M);
wr32(hw, VPINT_ALLOC_PCI(vf->vf_id), reg);
/* map the interrupts to its functions */
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
index ada69120ff38..424bc0538956 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
@@ -45,7 +45,8 @@ struct ice_vf {
s16 vf_id; /* VF ID in the PF space */
u16 lan_vsi_idx; /* index into PF struct */
- int first_vector_idx; /* first vector index of this VF */
+ /* first vector index of this VF in the PF space */
+ int first_vector_idx;
struct ice_sw *vf_sw_id; /* switch ID the VF VSIs connect to */
struct virtchnl_version_info vf_ver;
u32 driver_caps; /* reported by VF driver */
--
2.21.0
^ permalink raw reply related
* [net-next 09/15] ice: Do not always bring up PF VSI in ice_ena_vsi()
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem; +Cc: Tony Nguyen, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Tony Nguyen <anthony.l.nguyen@intel.com>
During rebuild ice_ena_vsi() is called to recover the VSI state.
This function assumes the PF VSI is always to be enabled, however,
it's possible that during reset/rebuild the interface can be
brought down. If this occurs, we can attempt to bring up the PF
VSI on a downed interface which can lead to various crashes. If
the interface is not running, do not bring up the associated VSI.
Signed-off-by: Tony Nguyen <anthony.l.nguyen@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice_main.c | 2 --
1 file changed, 2 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index f6926cbb48a4..8407eeafbc41 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -3737,8 +3737,6 @@ static int ice_ena_vsi(struct ice_vsi *vsi, bool locked)
err = netd->netdev_ops->ndo_open(netd);
rtnl_unlock();
}
- } else {
- err = ice_vsi_open(vsi);
}
}
--
2.21.0
^ permalink raw reply related
* [net-next 04/15] ice: Restructure VFs initialization flows
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem
Cc: Akeem G Abodunrin, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Akeem G Abodunrin <akeem.g.abodunrin@intel.com>
This patch restructures how VFs are configured, and resources allocated.
Instead of freeing resources that were never allocated, and resetting
empty VFs that have never been created - the new flow will just allocate
resources for number of requested VFs based on the availability.
During VFs initialization process, global interrupt is disabled, and
rearmed after getting MSIX vectors for VFs. This allows immediate mailbox
communications, instead of delaying it till later and VFs.
PF communications resulted to using polling instead of actual interrupt.
The issue manifested when creating higher number of VFs (128 VFs) per PF.
Signed-off-by: Akeem G Abodunrin <akeem.g.abodunrin@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice.h | 1 +
.../net/ethernet/intel/ice/ice_virtchnl_pf.c | 69 +++++++++++++------
2 files changed, 48 insertions(+), 22 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 07950ac4869f..112bdb662ea2 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -220,6 +220,7 @@ enum ice_state {
__ICE_CFG_BUSY,
__ICE_SERVICE_SCHED,
__ICE_SERVICE_DIS,
+ __ICE_OICR_INTR_DIS, /* Global OICR interrupt disabled */
__ICE_STATE_NBITS /* must be last */
};
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
index ce01cbe70ea4..93b835a24bb1 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
@@ -974,6 +974,47 @@ ice_vf_set_vsi_promisc(struct ice_vf *vf, struct ice_vsi *vsi, u8 promisc_m,
return status;
}
+/**
+ * ice_config_res_vfs - Finalize allocation of VFs resources in one go
+ * @pf: pointer to the PF structure
+ *
+ * This function is being called as last part of resetting all VFs, or when
+ * configuring VFs for the first time, where there is no resource to be freed
+ * Returns true if resources were properly allocated for all VFs, and false
+ * otherwise.
+ */
+static bool ice_config_res_vfs(struct ice_pf *pf)
+{
+ struct ice_hw *hw = &pf->hw;
+ int v;
+
+ if (ice_check_avail_res(pf)) {
+ dev_err(&pf->pdev->dev,
+ "Cannot allocate VF resources, try with fewer number of VFs\n");
+ return false;
+ }
+
+ /* rearm global interrupts */
+ if (test_and_clear_bit(__ICE_OICR_INTR_DIS, pf->state))
+ ice_irq_dynamic_ena(hw, NULL, NULL);
+
+ /* Finish resetting each VF and allocate resources */
+ for (v = 0; v < pf->num_alloc_vfs; v++) {
+ struct ice_vf *vf = &pf->vf[v];
+
+ vf->num_vf_qs = pf->num_vf_qps;
+ dev_dbg(&pf->pdev->dev,
+ "VF-id %d has %d queues configured\n",
+ vf->vf_id, vf->num_vf_qs);
+ ice_cleanup_and_realloc_vf(vf);
+ }
+
+ ice_flush(hw);
+ clear_bit(__ICE_VF_DIS, pf->state);
+
+ return true;
+}
+
/**
* ice_reset_all_vfs - reset all allocated VFs in one go
* @pf: pointer to the PF structure
@@ -1066,25 +1107,8 @@ bool ice_reset_all_vfs(struct ice_pf *pf, bool is_vflr)
dev_err(&pf->pdev->dev,
"Failed to free MSIX resources used by SR-IOV\n");
- if (ice_check_avail_res(pf)) {
- dev_err(&pf->pdev->dev,
- "Cannot allocate VF resources, try with fewer number of VFs\n");
+ if (!ice_config_res_vfs(pf))
return false;
- }
-
- /* Finish the reset on each VF */
- for (v = 0; v < pf->num_alloc_vfs; v++) {
- vf = &pf->vf[v];
-
- vf->num_vf_qs = pf->num_vf_qps;
- dev_dbg(&pf->pdev->dev,
- "VF-id %d has %d queues configured\n",
- vf->vf_id, vf->num_vf_qs);
- ice_cleanup_and_realloc_vf(vf);
- }
-
- ice_flush(hw);
- clear_bit(__ICE_VF_DIS, pf->state);
return true;
}
@@ -1249,7 +1273,7 @@ static int ice_alloc_vfs(struct ice_pf *pf, u16 num_alloc_vfs)
/* Disable global interrupt 0 so we don't try to handle the VFLR. */
wr32(hw, GLINT_DYN_CTL(pf->oicr_idx),
ICE_ITR_NONE << GLINT_DYN_CTL_ITR_INDX_S);
-
+ set_bit(__ICE_OICR_INTR_DIS, pf->state);
ice_flush(hw);
ret = pci_enable_sriov(pf->pdev, num_alloc_vfs);
@@ -1278,13 +1302,13 @@ static int ice_alloc_vfs(struct ice_pf *pf, u16 num_alloc_vfs)
}
pf->num_alloc_vfs = num_alloc_vfs;
- /* VF resources get allocated during reset */
- if (!ice_reset_all_vfs(pf, true)) {
+ /* VF resources get allocated with initialization */
+ if (!ice_config_res_vfs(pf)) {
ret = -EIO;
goto err_unroll_sriov;
}
- goto err_unroll_intr;
+ return ret;
err_unroll_sriov:
pf->vf = NULL;
@@ -1296,6 +1320,7 @@ static int ice_alloc_vfs(struct ice_pf *pf, u16 num_alloc_vfs)
err_unroll_intr:
/* rearm interrupts here */
ice_irq_dynamic_ena(hw, NULL, NULL);
+ clear_bit(__ICE_OICR_INTR_DIS, pf->state);
return ret;
}
--
2.21.0
^ permalink raw reply related
* [net-next 14/15] ice: Change type for queue counts
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem
Cc: Pawel Kaminski, netdev, nhorman, sassmann, Tony Nguyen,
Andrew Bowers, Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Pawel Kaminski <pawel.kaminski@intel.com>
These queue variables are being assigned values that are type u16.
Change the local variables to match these types. Since these
represent queue counts, they should never be negative.
Signed-off-by: Pawel Kaminski <pawel.kaminski@intel.com>
Signed-off-by: Tony Nguyen <anthony.l.nguyen@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
.../net/ethernet/intel/ice/ice_virtchnl_pf.c | 25 ++++++++++---------
1 file changed, 13 insertions(+), 12 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
index 8a5bf9730fdf..61cf1c5af928 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
@@ -2337,11 +2337,11 @@ static int ice_vc_request_qs_msg(struct ice_vf *vf, u8 *msg)
enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS;
struct virtchnl_vf_res_request *vfres =
(struct virtchnl_vf_res_request *)msg;
- int req_queues = vfres->num_queue_pairs;
+ u16 req_queues = vfres->num_queue_pairs;
struct ice_pf *pf = vf->pf;
- int max_allowed_vf_queues;
- int tx_rx_queue_left;
- int cur_queues;
+ u16 max_allowed_vf_queues;
+ u16 tx_rx_queue_left;
+ u16 cur_queues;
if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) {
v_ret = VIRTCHNL_STATUS_ERR_PARAM;
@@ -2349,29 +2349,30 @@ static int ice_vc_request_qs_msg(struct ice_vf *vf, u8 *msg)
}
cur_queues = vf->num_vf_qs;
- tx_rx_queue_left = min_t(int, pf->q_left_tx, pf->q_left_rx);
+ tx_rx_queue_left = min_t(u16, pf->q_left_tx, pf->q_left_rx);
max_allowed_vf_queues = tx_rx_queue_left + cur_queues;
- if (req_queues <= 0) {
+ if (!req_queues) {
dev_err(&pf->pdev->dev,
- "VF %d tried to request %d queues. Ignoring.\n",
- vf->vf_id, req_queues);
+ "VF %d tried to request 0 queues. Ignoring.\n",
+ vf->vf_id);
} else if (req_queues > ICE_MAX_BASE_QS_PER_VF) {
dev_err(&pf->pdev->dev,
"VF %d tried to request more than %d queues.\n",
vf->vf_id, ICE_MAX_BASE_QS_PER_VF);
vfres->num_queue_pairs = ICE_MAX_BASE_QS_PER_VF;
- } else if (req_queues - cur_queues > tx_rx_queue_left) {
+ } else if (req_queues > cur_queues &&
+ req_queues - cur_queues > tx_rx_queue_left) {
dev_warn(&pf->pdev->dev,
- "VF %d requested %d more queues, but only %d left.\n",
+ "VF %d requested %u more queues, but only %u left.\n",
vf->vf_id, req_queues - cur_queues, tx_rx_queue_left);
- vfres->num_queue_pairs = min_t(int, max_allowed_vf_queues,
+ vfres->num_queue_pairs = min_t(u16, max_allowed_vf_queues,
ICE_MAX_BASE_QS_PER_VF);
} else {
/* request is successful, then reset VF */
vf->num_req_qs = req_queues;
ice_vc_dis_vf(vf);
dev_info(&pf->pdev->dev,
- "VF %d granted request of %d queues.\n",
+ "VF %d granted request of %u queues.\n",
vf->vf_id, req_queues);
return 0;
}
--
2.21.0
^ permalink raw reply related
* [net-next 13/15] ice: Move VF resources definition to SR-IOV specific file
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem
Cc: Akeem G Abodunrin, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Akeem G Abodunrin <akeem.g.abodunrin@intel.com>
In order to use some of the VF resources definition in the SR-IOV specific
virtchnl header file, this patch moves applicable code to
ice_virtchnl_pf.h file accordingly... and they should have been defined in
the destination file originally.
Signed-off-by: Akeem G Abodunrin <akeem.g.abodunrin@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice.h | 10 ----------
drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h | 13 +++++++++++++
2 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 36a8ee873ae1..8398dd45a8a2 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -87,16 +87,6 @@ extern const char ice_drv_ver[];
#define ICE_RES_MISC_VEC_ID (ICE_RES_VALID_BIT - 1)
#define ICE_INVAL_Q_INDEX 0xffff
#define ICE_INVAL_VFID 256
-#define ICE_MAX_VF_COUNT 256
-#define ICE_MAX_QS_PER_VF 256
-#define ICE_MIN_QS_PER_VF 1
-#define ICE_DFLT_QS_PER_VF 4
-#define ICE_NONQ_VECS_VF 1
-#define ICE_MAX_SCATTER_QS_PER_VF 16
-#define ICE_MAX_BASE_QS_PER_VF 16
-#define ICE_MAX_INTR_PER_VF 65
-#define ICE_MIN_INTR_PER_VF (ICE_MIN_QS_PER_VF + 1)
-#define ICE_DFLT_INTR_PER_VF (ICE_DFLT_QS_PER_VF + 1)
#define ICE_MAX_RESET_WAIT 20
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
index 79bb47f73879..4d94853f119a 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
@@ -26,6 +26,19 @@
#define ICE_PCI_CIAD_WAIT_COUNT 100
#define ICE_PCI_CIAD_WAIT_DELAY_US 1
+/* VF resources default values and limitation */
+#define ICE_MAX_VF_COUNT 256
+#define ICE_MAX_QS_PER_VF 256
+#define ICE_MIN_QS_PER_VF 1
+#define ICE_DFLT_QS_PER_VF 4
+#define ICE_NONQ_VECS_VF 1
+#define ICE_MAX_SCATTER_QS_PER_VF 16
+#define ICE_MAX_BASE_QS_PER_VF 16
+#define ICE_MAX_INTR_PER_VF 65
+#define ICE_MAX_POLICY_INTR_PER_VF 33
+#define ICE_MIN_INTR_PER_VF (ICE_MIN_QS_PER_VF + 1)
+#define ICE_DFLT_INTR_PER_VF (ICE_DFLT_QS_PER_VF + 1)
+
/* Specific VF states */
enum ice_vf_states {
ICE_VF_STATE_INIT = 0,
--
2.21.0
^ permalink raw reply related
* [net-next 12/15] ice: Increase size of Mailbox receive queue for many VFs
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem; +Cc: Brett Creeley, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Brett Creeley <brett.creeley@intel.com>
Currently we use the ICE_MBXQ_LEN for both the Mailbox send and receive
queues that are used to communicate with VFs. This is fine for the send
queue because the PF driver will lock the queue for every single send,
but for the Mailbox receive queue every VF is posting to its Mailbox
send queue and the hardware is then handing the message to the PF on its
Mailbox receive queue. This becomes a problem with many VFs because it
seems to overburden the Mailbox receive queue on the PF. Fix this by
increasing the Mailbox receive queue for the PF to 512 entries.
The number 512 was determined based on the number of VFs supported by
the device. We can have a total of 256 VFs so in the worst case this
allows the VFs to put 2 messages in the PFs Mailbox receive queue at the
same time.
Signed-off-by: Brett Creeley <brett.creeley@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice.h | 3 ++-
drivers/net/ethernet/intel/ice/ice_main.c | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 112bdb662ea2..36a8ee873ae1 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -69,7 +69,8 @@ extern const char ice_drv_ver[];
#define ICE_INT_NAME_STR_LEN (IFNAMSIZ + 16)
#define ICE_ETHTOOL_FWVER_LEN 32
#define ICE_AQ_LEN 64
-#define ICE_MBXQ_LEN 64
+#define ICE_MBXSQ_LEN 64
+#define ICE_MBXRQ_LEN 512
#define ICE_MIN_MSIX 2
#define ICE_NO_VSI 0xffff
#define ICE_MAX_TXQS 2048
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index 8407eeafbc41..1d406b8abf5a 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -1507,8 +1507,8 @@ static void ice_set_ctrlq_len(struct ice_hw *hw)
hw->adminq.num_sq_entries = ICE_AQ_LEN;
hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;
- hw->mailboxq.num_rq_entries = ICE_MBXQ_LEN;
- hw->mailboxq.num_sq_entries = ICE_MBXQ_LEN;
+ hw->mailboxq.num_rq_entries = ICE_MBXRQ_LEN;
+ hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN;
hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
}
--
2.21.0
^ permalink raw reply related
* [net-next 15/15] ice: improve print for VF's when adding/deleting MAC filters
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem; +Cc: Brett Creeley, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Brett Creeley <brett.creeley@intel.com>
When we fail to add/delete MAC filters in the VF, the print doesn't
distinguish between the two. Fix that by printing whether or not we
failed to add/delete the MAC filter respectively.
Signed-off-by: Brett Creeley <brett.creeley@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
index 61cf1c5af928..1b1d1ea0c8f9 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
@@ -2283,8 +2283,8 @@ ice_vc_handle_mac_addr_msg(struct ice_vf *vf, u8 *msg, bool set)
if (v_ret) {
dev_err(&pf->pdev->dev,
- "can't update MAC filters for VF %d, error %d\n",
- vf->vf_id, v_ret);
+ "can't %s MAC filters for VF %d, error %d\n",
+ set ? "add" : "remove", vf->vf_id, v_ret);
} else {
if (set)
vf->num_mac += mac_count;
--
2.21.0
^ permalink raw reply related
* [net-next 11/15] ice: Reduce wait times during VF bringup/reset
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem; +Cc: Brett Creeley, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Brett Creeley <brett.creeley@intel.com>
Currently there are a couple places where the VF is waiting too long when
checking the status of registers. This is causing the AVF driver to
spin for longer than necessary in the __IAVF_STARTUP state. Sometimes
it causes the AVF to go into the __IAVF_COMM_FAILED, which may retrigger
the __IAVF_STARTUP state. Try to reduce the chance of this happening by
removing unnecessary wait times in VF bringup/resets.
Signed-off-by: Brett Creeley <brett.creeley@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
.../net/ethernet/intel/ice/ice_virtchnl_pf.c | 26 +++++++++++--------
.../net/ethernet/intel/ice/ice_virtchnl_pf.h | 4 +++
2 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
index 54b0e88af4ca..8a5bf9730fdf 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c
@@ -382,12 +382,15 @@ static void ice_trigger_vf_reset(struct ice_vf *vf, bool is_vflr)
wr32(hw, PF_PCI_CIAA,
VF_DEVICE_STATUS | (vf_abs_id << PF_PCI_CIAA_VF_NUM_S));
- for (i = 0; i < 100; i++) {
+ for (i = 0; i < ICE_PCI_CIAD_WAIT_COUNT; i++) {
reg = rd32(hw, PF_PCI_CIAD);
- if ((reg & VF_TRANS_PENDING_M) != 0)
- dev_err(&pf->pdev->dev,
- "VF %d PCI transactions stuck\n", vf->vf_id);
- udelay(1);
+ /* no transactions pending so stop polling */
+ if ((reg & VF_TRANS_PENDING_M) == 0)
+ break;
+
+ dev_err(&pf->pdev->dev,
+ "VF %d PCI transactions stuck\n", vf->vf_id);
+ udelay(ICE_PCI_CIAD_WAIT_DELAY_US);
}
}
@@ -1068,7 +1071,6 @@ bool ice_reset_all_vfs(struct ice_pf *pf, bool is_vflr)
* finished resetting.
*/
for (i = 0, v = 0; i < 10 && v < pf->num_alloc_vfs; i++) {
- usleep_range(10000, 20000);
/* Check each VF in sequence */
while (v < pf->num_alloc_vfs) {
@@ -1076,8 +1078,11 @@ bool ice_reset_all_vfs(struct ice_pf *pf, bool is_vflr)
vf = &pf->vf[v];
reg = rd32(hw, VPGEN_VFRSTAT(vf->vf_id));
- if (!(reg & VPGEN_VFRSTAT_VFRD_M))
+ if (!(reg & VPGEN_VFRSTAT_VFRD_M)) {
+ /* only delay if the check failed */
+ usleep_range(10, 20);
break;
+ }
/* If the current VF has finished resetting, move on
* to the next VF in sequence.
@@ -1091,7 +1096,6 @@ bool ice_reset_all_vfs(struct ice_pf *pf, bool is_vflr)
*/
if (v < pf->num_alloc_vfs)
dev_warn(&pf->pdev->dev, "VF reset check timeout\n");
- usleep_range(10000, 20000);
/* free VF resources to begin resetting the VSI state */
for (v = 0; v < pf->num_alloc_vfs; v++) {
@@ -1165,12 +1169,14 @@ static bool ice_reset_vf(struct ice_vf *vf, bool is_vflr)
* poll the status register to make sure that the reset
* completed successfully.
*/
- usleep_range(10000, 20000);
reg = rd32(hw, VPGEN_VFRSTAT(vf->vf_id));
if (reg & VPGEN_VFRSTAT_VFRD_M) {
rsd = true;
break;
}
+
+ /* only sleep if the reset is not done */
+ usleep_range(10, 20);
}
/* Display a warning if VF didn't manage to reset in time, but need to
@@ -1180,8 +1186,6 @@ static bool ice_reset_vf(struct ice_vf *vf, bool is_vflr)
dev_warn(&pf->pdev->dev, "VF reset check timeout on VF %d\n",
vf->vf_id);
- usleep_range(10000, 20000);
-
/* disable promiscuous modes in case they were enabled
* ignore any error if disabling process failed
*/
diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
index 424bc0538956..79bb47f73879 100644
--- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
+++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h
@@ -22,6 +22,10 @@
#define VF_DEVICE_STATUS 0xAA
#define VF_TRANS_PENDING_M 0x20
+/* wait defines for polling PF_PCI_CIAD register status */
+#define ICE_PCI_CIAD_WAIT_COUNT 100
+#define ICE_PCI_CIAD_WAIT_DELAY_US 1
+
/* Specific VF states */
enum ice_vf_states {
ICE_VF_STATE_INIT = 0,
--
2.21.0
^ permalink raw reply related
* [net-next 08/15] ice: allow empty Rx descriptors
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem
Cc: Mitch Williams, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Mitch Williams <mitch.a.williams@intel.com>
In some circumstances, the hardware will hand us a receive descriptor
which has no data attached, but is otherwise valid. The receive code was
improperly ignoring these descriptors, which result in an infinite loop.
To fix this, change the receive code to process all descriptors,
regardless of the size of the associated data. Add checks to the
memory-handling functions to allow for zero size.
Signed-off-by: Mitch Williams <mitch.a.williams@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice_txrx.c | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c
index c88e0701e1d7..e5c4c9139e54 100644
--- a/drivers/net/ethernet/intel/ice/ice_txrx.c
+++ b/drivers/net/ethernet/intel/ice/ice_txrx.c
@@ -607,6 +607,8 @@ ice_add_rx_frag(struct ice_rx_buf *rx_buf, struct sk_buff *skb,
unsigned int truesize = ICE_RXBUF_2048;
#endif
+ if (!size)
+ return;
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page,
rx_buf->page_offset, size, truesize);
@@ -662,6 +664,8 @@ ice_get_rx_buf(struct ice_ring *rx_ring, struct sk_buff **skb,
prefetchw(rx_buf->page);
*skb = rx_buf->skb;
+ if (!size)
+ return rx_buf;
/* we are reusing so sync this buffer for CPU use */
dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma,
rx_buf->page_offset, size,
@@ -745,8 +749,11 @@ ice_construct_skb(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,
*/
static void ice_put_rx_buf(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
{
- /* hand second half of page back to the ring */
+ if (!rx_buf)
+ return;
+
if (ice_can_reuse_rx_page(rx_buf)) {
+ /* hand second half of page back to the ring */
ice_reuse_rx_page(rx_ring, rx_buf);
rx_ring->rx_stats.page_reuse_count++;
} else {
@@ -1031,8 +1038,9 @@ static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget)
size = le16_to_cpu(rx_desc->wb.pkt_len) &
ICE_RX_FLX_DESC_PKT_LEN_M;
+ /* retrieve a buffer from the ring */
rx_buf = ice_get_rx_buf(rx_ring, &skb, size);
- /* allocate (if needed) and populate skb */
+
if (skb)
ice_add_rx_frag(rx_buf, skb, size);
else
@@ -1041,7 +1049,8 @@ static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget)
/* exit if we failed to retrieve a buffer */
if (!skb) {
rx_ring->rx_stats.alloc_buf_failed++;
- rx_buf->pagecnt_bias++;
+ if (rx_buf)
+ rx_buf->pagecnt_bias++;
break;
}
--
2.21.0
^ permalink raw reply related
* [net-next 02/15] ice: Use the software based tail when checking for hung Tx ring
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem; +Cc: Brett Creeley, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Brett Creeley <brett.creeley@intel.com>
Currently in ice_get_tx_pending we try to read a Tx ring's tail. This is
then compared with the software based head (next_to_clean) to determine
if we have pending work. This will never work because reading of the Tx
ring's tail is no longer supported. Fix this by using the software based
tail (next_to_use) to determine if there is pending work.
Signed-off-by: Brett Creeley <brett.creeley@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice_main.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index f9f81307dac8..f6926cbb48a4 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -41,12 +41,12 @@ static void ice_update_pf_stats(struct ice_pf *pf);
* ice_get_tx_pending - returns number of Tx descriptors not processed
* @ring: the ring of descriptors
*/
-static u32 ice_get_tx_pending(struct ice_ring *ring)
+static u16 ice_get_tx_pending(struct ice_ring *ring)
{
- u32 head, tail;
+ u16 head, tail;
head = ring->next_to_clean;
- tail = readl(ring->tail);
+ tail = ring->next_to_use;
if (head != tail)
return (head < tail) ?
--
2.21.0
^ permalink raw reply related
* [net-next 01/15] ice: Implement ethtool ops for channels
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem
Cc: Henry Tieman, netdev, nhorman, sassmann, Tony Nguyen,
Andrew Bowers, Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Henry Tieman <henry.w.tieman@intel.com>
Add code to query and set the number of queues on the primary
VSI for a PF. This is accessed from the 'ethtool -l' and 'ethtool -L'
commands, respectively.
Signed-off-by: Henry Tieman <henry.w.tieman@intel.com>
Signed-off-by: Tony Nguyen <anthony.l.nguyen@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice.h | 4 +
drivers/net/ethernet/intel/ice/ice_dcb_lib.c | 9 +-
drivers/net/ethernet/intel/ice/ice_dcb_lib.h | 2 +
drivers/net/ethernet/intel/ice/ice_ethtool.c | 85 ++++++++++++++++++
drivers/net/ethernet/intel/ice/ice_lib.c | 52 ++++++++---
drivers/net/ethernet/intel/ice/ice_lib.h | 2 +-
drivers/net/ethernet/intel/ice/ice_main.c | 91 +++++++++++++++++++-
7 files changed, 230 insertions(+), 15 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 794d97460fc7..07950ac4869f 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -298,6 +298,8 @@ struct ice_vsi {
u16 num_txq; /* Used Tx queues */
u16 alloc_rxq; /* Allocated Rx queues */
u16 num_rxq; /* Used Rx queues */
+ u16 req_txq; /* User requested Tx queues */
+ u16 req_rxq; /* User requested Rx queues */
u16 num_rx_desc;
u16 num_tx_desc;
struct ice_tc_cfg tc_cfg;
@@ -455,6 +457,7 @@ ice_find_vsi_by_type(struct ice_pf *pf, enum ice_vsi_type type)
int ice_vsi_setup_tx_rings(struct ice_vsi *vsi);
int ice_vsi_setup_rx_rings(struct ice_vsi *vsi);
void ice_set_ethtool_ops(struct net_device *netdev);
+int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx);
int ice_up(struct ice_vsi *vsi);
int ice_down(struct ice_vsi *vsi);
int ice_vsi_cfg(struct ice_vsi *vsi);
@@ -462,6 +465,7 @@ struct ice_vsi *ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi);
int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size);
int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size);
void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size);
+int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset);
void ice_print_link_msg(struct ice_vsi *vsi, bool isup);
#ifdef CONFIG_DCB
int ice_pf_ena_all_vsi(struct ice_pf *pf, bool locked);
diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
index fe88b127ca42..f80628a13f2a 100644
--- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
@@ -85,11 +85,16 @@ void ice_vsi_cfg_dcb_rings(struct ice_vsi *vsi)
break;
qoffset = vsi->tc_cfg.tc_info[n].qoffset;
+
qcount = vsi->tc_cfg.tc_info[n].qcount_tx;
for (i = qoffset; i < (qoffset + qcount); i++) {
tx_ring = vsi->tx_rings[i];
- rx_ring = vsi->rx_rings[i];
tx_ring->dcb_tc = n;
+ }
+
+ qcount = vsi->tc_cfg.tc_info[n].qcount_rx;
+ for (i = qoffset; i < (qoffset + qcount); i++) {
+ rx_ring = vsi->rx_rings[i];
rx_ring->dcb_tc = n;
}
}
@@ -103,7 +108,7 @@ void ice_vsi_cfg_dcb_rings(struct ice_vsi *vsi)
* calling this function. Reconfiguring DCB based on
* local_dcbx_cfg.
*/
-static void ice_pf_dcb_recfg(struct ice_pf *pf)
+void ice_pf_dcb_recfg(struct ice_pf *pf)
{
struct ice_dcbx_cfg *dcbcfg = &pf->hw.port_info->local_dcbx_cfg;
u8 tc_map = 0;
diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.h b/drivers/net/ethernet/intel/ice/ice_dcb_lib.h
index 819081053ff5..be2ab6f20b21 100644
--- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.h
+++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.h
@@ -13,6 +13,7 @@
void ice_dcb_rebuild(struct ice_pf *pf);
u8 ice_dcb_get_ena_tc(struct ice_dcbx_cfg *dcbcfg);
u8 ice_dcb_get_num_tc(struct ice_dcbx_cfg *dcbcfg);
+void ice_pf_dcb_recfg(struct ice_pf *pf);
void ice_vsi_cfg_dcb_rings(struct ice_vsi *vsi);
int ice_init_pf_dcb(struct ice_pf *pf, bool locked);
void ice_update_dcb_stats(struct ice_pf *pf);
@@ -55,6 +56,7 @@ ice_tx_prepare_vlan_flags_dcb(struct ice_ring __always_unused *tx_ring,
}
#define ice_update_dcb_stats(pf) do {} while (0)
+#define ice_pf_dcb_recfg(pf) do {} while (0)
#define ice_vsi_cfg_dcb_rings(vsi) do {} while (0)
#define ice_dcb_process_lldp_set_mib_change(pf, event) do {} while (0)
#define ice_set_cgd_num(tlan_ctx, ring) do {} while (0)
diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c
index d3ba535bd65a..1fe048cfb737 100644
--- a/drivers/net/ethernet/intel/ice/ice_ethtool.c
+++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c
@@ -3083,6 +3083,89 @@ ice_set_rxfh(struct net_device *netdev, const u32 *indir, const u8 *key,
return 0;
}
+/**
+ * ice_get_max_txq - return the maximum number of Tx queues for in a PF
+ * @pf: PF structure
+ */
+static int ice_get_max_txq(struct ice_pf *pf)
+{
+ return min_t(int, num_online_cpus(),
+ pf->hw.func_caps.common_cap.num_txq);
+}
+
+/**
+ * ice_get_max_rxq - return the maximum number of Rx queues for in a PF
+ * @pf: PF structure
+ */
+static int ice_get_max_rxq(struct ice_pf *pf)
+{
+ return min_t(int, num_online_cpus(),
+ pf->hw.func_caps.common_cap.num_rxq);
+}
+
+/**
+ * ice_get_channels - get the current and max supported channels
+ * @dev: network interface device structure
+ * @ch: ethtool channel data structure
+ */
+static void
+ice_get_channels(struct net_device *dev, struct ethtool_channels *ch)
+{
+ struct ice_netdev_priv *np = netdev_priv(dev);
+ struct ice_vsi *vsi = np->vsi;
+ struct ice_pf *pf = vsi->back;
+
+ /* check to see if VSI is active */
+ if (test_bit(__ICE_DOWN, vsi->state))
+ return;
+
+ /* report maximum channels */
+ ch->max_rx = ice_get_max_rxq(pf);
+ ch->max_tx = ice_get_max_txq(pf);
+
+ /* report current channels */
+ ch->rx_count = vsi->num_rxq;
+ ch->tx_count = vsi->num_txq;
+}
+
+/**
+ * ice_set_channels - set the number channels
+ * @dev: network interface device structure
+ * @ch: ethtool channel data structure
+ */
+static int ice_set_channels(struct net_device *dev, struct ethtool_channels *ch)
+{
+ struct ice_netdev_priv *np = netdev_priv(dev);
+ struct ice_vsi *vsi = np->vsi;
+ struct ice_pf *pf = vsi->back;
+ int new_rx = 0, new_tx = 0;
+
+ /* do not support changing other_count */
+ if (ch->other_count)
+ return -EINVAL;
+
+ /* verify request for a valid number of channels */
+ if (ch->rx_count > ice_get_max_rxq(pf) ||
+ ch->tx_count > ice_get_max_txq(pf))
+ return -EINVAL;
+
+ /* Use new Rx value only if changed */
+ if (ch->rx_count != vsi->num_rxq)
+ new_rx = ch->rx_count;
+
+ /* Use new Tx value only if changed */
+ if (ch->tx_count != vsi->num_txq)
+ new_tx = ch->tx_count;
+
+ /* verify that we have a valid request */
+ if (!new_rx && !new_tx)
+ return -EINVAL;
+
+ ice_vsi_recfg_qs(vsi, new_rx, new_tx);
+
+ return 0;
+}
+
enum ice_container_type {
ICE_RX_CONTAINER,
ICE_TX_CONTAINER,
@@ -3429,6 +3512,8 @@ static const struct ethtool_ops ice_ethtool_ops = {
.get_rxfh_indir_size = ice_get_rxfh_indir_size,
.get_rxfh = ice_get_rxfh,
.set_rxfh = ice_set_rxfh,
+ .get_channels = ice_get_channels,
+ .set_channels = ice_set_channels,
.get_ts_info = ethtool_op_get_ts_info,
.get_per_queue_coalesce = ice_get_per_q_coalesce,
.set_per_queue_coalesce = ice_set_per_q_coalesce,
diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
index 6e34c40e7840..a62496a74a69 100644
--- a/drivers/net/ethernet/intel/ice/ice_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_lib.c
@@ -313,6 +313,14 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi, u16 vf_id)
case ICE_VSI_PF:
vsi->alloc_txq = pf->num_lan_tx;
vsi->alloc_rxq = pf->num_lan_rx;
+ if (vsi->req_txq) {
+ vsi->alloc_txq = vsi->req_txq;
+ vsi->num_txq = vsi->req_txq;
+ }
+ if (vsi->req_rxq) {
+ vsi->alloc_rxq = vsi->req_rxq;
+ vsi->num_rxq = vsi->req_rxq;
+ }
vsi->num_q_vectors = max_t(int, pf->num_lan_rx, pf->num_lan_tx);
break;
case ICE_VSI_VF:
@@ -855,7 +863,9 @@ static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt)
else
max_rss = ICE_MAX_SMALL_RSS_QS;
qcount_rx = min_t(int, rx_numq_tc, max_rss);
- qcount_rx = min_t(int, qcount_rx, vsi->rss_size);
+ if (!vsi->req_rxq)
+ qcount_rx = min_t(int, qcount_rx,
+ vsi->rss_size);
}
}
@@ -959,11 +969,12 @@ static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi)
/**
* ice_vsi_init - Create and initialize a VSI
* @vsi: the VSI being configured
+ * @init_vsi: is this call creating a VSI
*
* This initializes a VSI context depending on the VSI type to be added and
* passes it down to the add_vsi aq command to create a new VSI.
*/
-static int ice_vsi_init(struct ice_vsi *vsi)
+static int ice_vsi_init(struct ice_vsi *vsi, bool init_vsi)
{
struct ice_pf *pf = vsi->back;
struct ice_hw *hw = &pf->hw;
@@ -1010,11 +1021,20 @@ static int ice_vsi_init(struct ice_vsi *vsi)
ICE_AQ_VSI_SEC_FLAG_ENA_MAC_ANTI_SPOOF;
}
- ret = ice_add_vsi(hw, vsi->idx, ctxt, NULL);
- if (ret) {
- dev_err(&pf->pdev->dev,
- "Add VSI failed, err %d\n", ret);
- return -EIO;
+ if (init_vsi) {
+ ret = ice_add_vsi(hw, vsi->idx, ctxt, NULL);
+ if (ret) {
+ dev_err(&pf->pdev->dev,
+ "Add VSI failed, err %d\n", ret);
+ return -EIO;
+ }
+ } else {
+ ret = ice_update_vsi(hw, vsi->idx, ctxt, NULL);
+ if (ret) {
+ dev_err(&pf->pdev->dev,
+ "Update VSI failed, err %d\n", ret);
+ return -EIO;
+ }
}
/* keep context for update VSI operations */
@@ -2435,7 +2455,7 @@ ice_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi,
ice_vsi_set_tc_cfg(vsi);
/* create the VSI */
- ret = ice_vsi_init(vsi);
+ ret = ice_vsi_init(vsi, true);
if (ret)
goto unroll_get_qs;
@@ -2911,10 +2931,11 @@ int ice_vsi_release(struct ice_vsi *vsi)
/**
* ice_vsi_rebuild - Rebuild VSI after reset
* @vsi: VSI to be rebuild
+ * @init_vsi: is this an initialization or a reconfigure of the VSI
*
* Returns 0 on success and negative value on failure
*/
-int ice_vsi_rebuild(struct ice_vsi *vsi)
+int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi)
{
u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
struct ice_vf *vf = NULL;
@@ -2947,14 +2968,18 @@ int ice_vsi_rebuild(struct ice_vsi *vsi)
ice_vsi_clear_rings(vsi);
ice_vsi_free_arrays(vsi);
ice_dev_onetime_setup(&pf->hw);
+ if (vsi->req_txq || vsi->req_rxq)
+ ice_vsi_put_qs(vsi);
if (vsi->type == ICE_VSI_VF)
ice_vsi_set_num_qs(vsi, vf->vf_id);
else
ice_vsi_set_num_qs(vsi, ICE_INVAL_VFID);
+ if (vsi->req_txq || vsi->req_rxq)
+ ice_vsi_get_qs(vsi);
ice_vsi_set_tc_cfg(vsi);
/* Initialize VSI struct elements and create VSI in FW */
- ret = ice_vsi_init(vsi);
+ ret = ice_vsi_init(vsi, init_vsi);
if (ret < 0)
goto err_vsi;
@@ -3018,7 +3043,12 @@ int ice_vsi_rebuild(struct ice_vsi *vsi)
dev_err(&pf->pdev->dev,
"VSI %d failed lan queue config, error %d\n",
vsi->vsi_num, status);
- goto err_vectors;
+ if (init_vsi) {
+ ret = -EIO;
+ goto err_vectors;
+ } else {
+ return ice_schedule_reset(pf, ICE_RESET_PFR);
+ }
}
return 0;
diff --git a/drivers/net/ethernet/intel/ice/ice_lib.h b/drivers/net/ethernet/intel/ice/ice_lib.h
index 6e43ef03bfc3..7409a69f631d 100644
--- a/drivers/net/ethernet/intel/ice/ice_lib.h
+++ b/drivers/net/ethernet/intel/ice/ice_lib.h
@@ -70,7 +70,7 @@ int ice_free_res(struct ice_res_tracker *res, u16 index, u16 id);
int
ice_get_res(struct ice_pf *pf, struct ice_res_tracker *res, u16 needed, u16 id);
-int ice_vsi_rebuild(struct ice_vsi *vsi);
+int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi);
bool ice_is_reset_in_progress(unsigned long *state);
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index c26e6a102dac..f9f81307dac8 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -1513,6 +1513,42 @@ static void ice_set_ctrlq_len(struct ice_hw *hw)
hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
}
+/**
+ * ice_schedule_reset - schedule a reset
+ * @pf: board private structure
+ * @reset: reset being requested
+ */
+int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset)
+{
+ /* bail out if earlier reset has failed */
+ if (test_bit(__ICE_RESET_FAILED, pf->state)) {
+ dev_dbg(&pf->pdev->dev, "earlier reset has failed\n");
+ return -EIO;
+ }
+ /* bail if reset/recovery already in progress */
+ if (ice_is_reset_in_progress(pf->state)) {
+ dev_dbg(&pf->pdev->dev, "Reset already in progress\n");
+ return -EBUSY;
+ }
+
+ switch (reset) {
+ case ICE_RESET_PFR:
+ set_bit(__ICE_PFR_REQ, pf->state);
+ break;
+ case ICE_RESET_CORER:
+ set_bit(__ICE_CORER_REQ, pf->state);
+ break;
+ case ICE_RESET_GLOBR:
+ set_bit(__ICE_GLOBR_REQ, pf->state);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ ice_service_task_schedule(pf);
+ return 0;
+}
+
/**
* ice_irq_affinity_notify - Callback for affinity changes
* @notify: context as to what irq was changed
@@ -3745,7 +3781,7 @@ static int ice_vsi_rebuild_all(struct ice_pf *pf)
if (!pf->vsi[i])
continue;
- err = ice_vsi_rebuild(pf->vsi[i]);
+ err = ice_vsi_rebuild(pf->vsi[i], true);
if (err) {
dev_err(&pf->pdev->dev,
"VSI at index %d rebuild failed\n",
@@ -3907,6 +3943,59 @@ static void ice_rebuild(struct ice_pf *pf)
dev_err(dev, "Rebuild failed, unload and reload driver\n");
}
+/**
+ * ice_vsi_recfg_qs - Change the number of queues on a VSI
+ * @vsi: VSI being changed
+ * @new_rx: new number of Rx queues
+ * @new_tx: new number of Tx queues
+ *
+ * Only change the number of queues if new_tx, or new_rx is non-0.
+ *
+ * Returns 0 on success.
+ */
+int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx)
+{
+ struct ice_pf *pf = vsi->back;
+ int err = 0, timeout = 50;
+
+ if (!new_rx && !new_tx)
+ return -EINVAL;
+
+ while (test_and_set_bit(__ICE_CFG_BUSY, pf->state)) {
+ timeout--;
+ if (!timeout)
+ return -EBUSY;
+ usleep_range(1000, 2000);
+ }
+
+ /* set for the next time the netdev is started */
+ if (!netif_running(vsi->netdev)) {
+ if (new_tx)
+ vsi->req_txq = new_tx;
+ if (new_rx)
+ vsi->req_rxq = new_rx;
+
+ dev_dbg(&pf->pdev->dev, "Link is down, queue count change happens when link is brought up\n");
+ goto done;
+ }
+
+ ice_vsi_close(vsi);
+
+ if (new_tx)
+ vsi->req_txq = new_tx;
+
+ if (new_rx)
+ vsi->req_rxq = new_rx;
+
+ ice_vsi_rebuild(vsi, false);
+ ice_pf_dcb_recfg(pf);
+
+ ice_vsi_open(vsi);
+done:
+ clear_bit(__ICE_CFG_BUSY, pf->state);
+ return err;
+}
+
/**
* ice_change_mtu - NDO callback to change the MTU
* @netdev: network interface device structure
--
2.21.0
^ permalink raw reply related
* [net-next 03/15] ice: Assume that more than one Rx queue is rare in ice_napi_poll
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem; +Cc: Brett Creeley, netdev, nhorman, sassmann, Andrew Bowers,
Jeff Kirsher
In-Reply-To: <20190809183139.30871-1-jeffrey.t.kirsher@intel.com>
From: Brett Creeley <brett.creeley@intel.com>
Currently we divide budget by the number of Rx queues per Rx ring
container in ice_napi_poll even if there is only 1. This is an
unnecessary divide for the normal case of 1 Rx ring per Rx ring
container. Fix this by using an unlikely() call in the case where we
actually need to divide.
Also, we will always set budget_per_ring even if there are no Rx rings
in the Rx ring container so we don't need to initialize it to 0.
Signed-off-by: Brett Creeley <brett.creeley@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
---
drivers/net/ethernet/intel/ice/ice_txrx.c | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c
index 9234fd203929..837d6ae2f33b 100644
--- a/drivers/net/ethernet/intel/ice/ice_txrx.c
+++ b/drivers/net/ethernet/intel/ice/ice_txrx.c
@@ -1414,8 +1414,8 @@ int ice_napi_poll(struct napi_struct *napi, int budget)
container_of(napi, struct ice_q_vector, napi);
struct ice_vsi *vsi = q_vector->vsi;
bool clean_complete = true;
- int budget_per_ring = 0;
struct ice_ring *ring;
+ int budget_per_ring;
int work_done = 0;
/* Since the actual Tx work is minimal, we can give the Tx a larger
@@ -1429,11 +1429,16 @@ int ice_napi_poll(struct napi_struct *napi, int budget)
if (budget <= 0)
return budget;
- /* We attempt to distribute budget to each Rx queue fairly, but don't
- * allow the budget to go below 1 because that would exit polling early.
- */
- if (q_vector->num_ring_rx)
+ /* normally we have 1 Rx ring per q_vector */
+ if (unlikely(q_vector->num_ring_rx > 1))
+ /* We attempt to distribute budget to each Rx queue fairly, but
+ * don't allow the budget to go below 1 because that would exit
+ * polling early.
+ */
budget_per_ring = max(budget / q_vector->num_ring_rx, 1);
+ else
+ /* Max of 1 Rx ring in this q_vector so give it the budget */
+ budget_per_ring = budget;
ice_for_each_ring(ring, q_vector->rx) {
int cleaned;
--
2.21.0
^ permalink raw reply related
* [net-next 00/15][pull request] 100GbE Intel Wired LAN Driver Updates 2019-08-09
From: Jeff Kirsher @ 2019-08-09 18:31 UTC (permalink / raw)
To: davem; +Cc: Jeff Kirsher, netdev, nhorman, sassmann
This series contains updates to ice driver only.
Henry adds support to query/add queues or channels on a VSI via ethtool
operations.
Brett fixes the detection of a hung transmit ring by checking the
software based tail (next_to_use) to determine if there is pending work.
Updates the driver to assume that using more than one receive queue per
receive ring container is a rare case, so use unlikely() in the case
were we actually need to divide our budget for multiple queues. Fixed
an issue where the write back on ITR bit was not being set when
interrupts are disabled, which was causing only write backs when polling
only when a cache line is filled. Cleans up unnecessary wait times
during VF bring up and reset paths. Increased the mailbox size for
receive queues that are used to communicate with VFs to accommodate the
large number of VFs that the driver can support.
Akeem restructures the initialization flows for VFs, including how VFs
are configured and resources allocated to improve flows so that when we
clean up resources, we do not try to free resources that were never
allocated. Organizes code to ensure that VF specific code is located in
the SR-IOV specific file.
Paul fixes an issue when setting the pause parameter which was
incorrectly blocking users from changing receive or transmit pause
settings. Ensure register access for MSIX vector index is only done in
the PF space and not absolute device space.
Usha fixes a potential kernel hang in the DCB rebuild path when in CEE
mode, where the ETS recommended DCB configuration is not being set or
set correctly.
Mitch updates the driver to process all receive descriptors, regardless
of the size of the associated data.
Tony fixes and issue during the reset/rebuild path of a PF VSI where we
were assuming that the PF VSI was always to be enabled, which can
attempt to bring up a PF VSI on a downed interface which can lead to
various crashes.
Pawel fixes up variable definitions to match the type of data being
stored.
The following are changes since commit fcc32a21655e26d30c746b4828b33a5fd4ccfb11:
liquidio: Use pcie_flr() instead of reimplementing it
and are available in the git repository at:
git://git.kernel.org/pub/scm/linux/kernel/git/jkirsher/next-queue 100GbE
Akeem G Abodunrin (2):
ice: Restructure VFs initialization flows
ice: Move VF resources definition to SR-IOV specific file
Brett Creeley (6):
ice: Use the software based tail when checking for hung Tx ring
ice: Assume that more than one Rx queue is rare in ice_napi_poll
ice: Set WB_ON_ITR when we don't re-enable interrupts
ice: Reduce wait times during VF bringup/reset
ice: Increase size of Mailbox receive queue for many VFs
ice: improve print for VF's when adding/deleting MAC filters
Henry Tieman (1):
ice: Implement ethtool ops for channels
Mitch Williams (1):
ice: allow empty Rx descriptors
Paul Greenwalt (2):
ice: fix set pause param autoneg check
ice: update GLINT_DYN_CTL and GLINT_VECT2FUNC register access
Pawel Kaminski (1):
ice: Change type for queue counts
Tony Nguyen (1):
ice: Do not always bring up PF VSI in ice_ena_vsi()
Usha Ketineni (1):
ice: Fix kernel hang with DCB reset in CEE mode
drivers/net/ethernet/intel/ice/ice.h | 18 +-
drivers/net/ethernet/intel/ice/ice_dcb_lib.c | 158 +++++++++++-------
drivers/net/ethernet/intel/ice/ice_dcb_lib.h | 2 +
drivers/net/ethernet/intel/ice/ice_ethtool.c | 113 ++++++++++++-
.../net/ethernet/intel/ice/ice_hw_autogen.h | 3 +
drivers/net/ethernet/intel/ice/ice_lib.c | 52 ++++--
drivers/net/ethernet/intel/ice/ice_lib.h | 2 +-
drivers/net/ethernet/intel/ice/ice_main.c | 103 +++++++++++-
drivers/net/ethernet/intel/ice/ice_txrx.c | 84 +++++++++-
drivers/net/ethernet/intel/ice/ice_txrx.h | 13 ++
.../net/ethernet/intel/ice/ice_virtchnl_pf.c | 148 +++++++++-------
.../net/ethernet/intel/ice/ice_virtchnl_pf.h | 20 ++-
12 files changed, 555 insertions(+), 161 deletions(-)
--
2.21.0
^ permalink raw reply
* Re: [PATCH net 0/2] rxrpc: Fixes
From: David Miller @ 2019-08-09 18:27 UTC (permalink / raw)
To: dhowells; +Cc: netdev, linux-afs, linux-kernel
In-Reply-To: <156536674651.17478.15139844428920800280.stgit@warthog.procyon.org.uk>
From: David Howells <dhowells@redhat.com>
Date: Fri, 09 Aug 2019 17:05:46 +0100
> Here's a couple of fixes for rxrpc:
>
> (1) Fix refcounting of the local endpoint.
>
> (2) Don't calculate or report packet skew information. This has been
> obsolete since AFS 3.1 and so is a waste of resources.
>
> The patches are tagged here:
>
> git://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git
> rxrpc-fixes-20190809
Pulled, thanks David.
^ permalink raw reply
* Re: [patch net-next] devlink: remove pointless data_len arg from region snapshot create
From: David Miller @ 2019-08-09 18:22 UTC (permalink / raw)
To: jiri; +Cc: netdev, tariqt, valex, mlxsw
In-Reply-To: <20190809132715.24282-1-jiri@resnulli.us>
From: Jiri Pirko <jiri@resnulli.us>
Date: Fri, 9 Aug 2019 15:27:15 +0200
> From: Jiri Pirko <jiri@mellanox.com>
>
> The size of the snapshot has to be the same as the size of the region,
> therefore no need to pass it again during snapshot creation. Remove the
> arg and use region->size instead.
>
> Signed-off-by: Jiri Pirko <jiri@mellanox.com>
Applied.
^ permalink raw reply
* Re: [PATCH v3 03/14] net: phy: adin: add support for interrupts
From: Heiner Kallweit @ 2019-08-09 18:19 UTC (permalink / raw)
To: Alexandru Ardelean, netdev, devicetree, linux-kernel
Cc: davem, robh+dt, mark.rutland, f.fainelli, andrew
In-Reply-To: <20190809133552.21597-4-alexandru.ardelean@analog.com>
On 09.08.2019 15:35, Alexandru Ardelean wrote:
> This change adds support for enabling PHY interrupts that can be used by
> the PHY framework to get signal for link/speed/auto-negotiation changes.
>
> Reviewed-by: Andrew Lunn <andrew@lunn.ch>
> Signed-off-by: Alexandru Ardelean <alexandru.ardelean@analog.com>
> ---
> drivers/net/phy/adin.c | 38 ++++++++++++++++++++++++++++++++++++++
> 1 file changed, 38 insertions(+)
>
> diff --git a/drivers/net/phy/adin.c b/drivers/net/phy/adin.c
> index fc0148ba4b94..91ff26d08fd5 100644
> --- a/drivers/net/phy/adin.c
> +++ b/drivers/net/phy/adin.c
> @@ -14,11 +14,45 @@
> #define PHY_ID_ADIN1200 0x0283bc20
> #define PHY_ID_ADIN1300 0x0283bc30
>
> +#define ADIN1300_INT_MASK_REG 0x0018
> +#define ADIN1300_INT_MDIO_SYNC_EN BIT(9)
> +#define ADIN1300_INT_ANEG_STAT_CHNG_EN BIT(8)
> +#define ADIN1300_INT_ANEG_PAGE_RX_EN BIT(6)
> +#define ADIN1300_INT_IDLE_ERR_CNT_EN BIT(5)
> +#define ADIN1300_INT_MAC_FIFO_OU_EN BIT(4)
> +#define ADIN1300_INT_RX_STAT_CHNG_EN BIT(3)
> +#define ADIN1300_INT_LINK_STAT_CHNG_EN BIT(2)
> +#define ADIN1300_INT_SPEED_CHNG_EN BIT(1)
> +#define ADIN1300_INT_HW_IRQ_EN BIT(0)
> +#define ADIN1300_INT_MASK_EN \
> + (ADIN1300_INT_ANEG_STAT_CHNG_EN | ADIN1300_INT_ANEG_PAGE_RX_EN | \
> + ADIN1300_INT_LINK_STAT_CHNG_EN | ADIN1300_INT_SPEED_CHNG_EN | \
> + ADIN1300_INT_HW_IRQ_EN)
phylib only needs the link status change interrupt. It shouldn't hurt
if enable more interrupt sources, but it's not needed.
> +#define ADIN1300_INT_STATUS_REG 0x0019
> +
> static int adin_config_init(struct phy_device *phydev)
> {
> return genphy_config_init(phydev);
> }
>
> +static int adin_phy_ack_intr(struct phy_device *phydev)
> +{
> + /* Clear pending interrupts */
> + int rc = phy_read(phydev, ADIN1300_INT_STATUS_REG);
> +
> + return rc < 0 ? rc : 0;
> +}
> +
> +static int adin_phy_config_intr(struct phy_device *phydev)
> +{
> + if (phydev->interrupts == PHY_INTERRUPT_ENABLED)
> + return phy_set_bits(phydev, ADIN1300_INT_MASK_REG,
> + ADIN1300_INT_MASK_EN);
> +
> + return phy_clear_bits(phydev, ADIN1300_INT_MASK_REG,
> + ADIN1300_INT_MASK_EN);
> +}
> +
> static struct phy_driver adin_driver[] = {
> {
> PHY_ID_MATCH_MODEL(PHY_ID_ADIN1200),
> @@ -26,6 +60,8 @@ static struct phy_driver adin_driver[] = {
> .config_init = adin_config_init,
> .config_aneg = genphy_config_aneg,
> .read_status = genphy_read_status,
> + .ack_interrupt = adin_phy_ack_intr,
> + .config_intr = adin_phy_config_intr,
> .resume = genphy_resume,
> .suspend = genphy_suspend,
> },
> @@ -35,6 +71,8 @@ static struct phy_driver adin_driver[] = {
> .config_init = adin_config_init,
> .config_aneg = genphy_config_aneg,
> .read_status = genphy_read_status,
> + .ack_interrupt = adin_phy_ack_intr,
> + .config_intr = adin_phy_config_intr,
> .resume = genphy_resume,
> .suspend = genphy_suspend,
> },
>
^ permalink raw reply
* Re: [PATCH v3 04/14] net: phy: adin: add {write,read}_mmd hooks
From: Heiner Kallweit @ 2019-08-09 18:21 UTC (permalink / raw)
To: Alexandru Ardelean, netdev, devicetree, linux-kernel
Cc: davem, robh+dt, mark.rutland, f.fainelli, andrew
In-Reply-To: <20190809133552.21597-5-alexandru.ardelean@analog.com>
On 09.08.2019 15:35, Alexandru Ardelean wrote:
> Both ADIN1200 & ADIN1300 support Clause 45 access for some registers.
> The Extended Management Interface (EMI) registers are accessible via both
> Clause 45 (at register MDIO_MMD_VEND1) and using Clause 22.
>
> However, the Clause 22 access for MMD regs differs from the standard one
> defined by 802.3. The ADIN PHYs use registers ExtRegPtr (0x0010) and
> ExtRegData (0x0011) to access Clause 45 & EMI registers.
>
> The indirect access is done via the following mechanism (for both R/W):
> 1. Write the address of the register in the ExtRegPtr
> 2. Read/write the value of the register (written at ExtRegPtr)
>
> This mechanism is needed to manage configuration of chip settings and to
> access EEE registers (via Clause 22).
>
> Signed-off-by: Alexandru Ardelean <alexandru.ardelean@analog.com>
> ---
> drivers/net/phy/adin.c | 46 ++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 46 insertions(+)
>
> diff --git a/drivers/net/phy/adin.c b/drivers/net/phy/adin.c
> index 91ff26d08fd5..8973ad819b93 100644
> --- a/drivers/net/phy/adin.c
> +++ b/drivers/net/phy/adin.c
> @@ -14,6 +14,9 @@
> #define PHY_ID_ADIN1200 0x0283bc20
> #define PHY_ID_ADIN1300 0x0283bc30
>
> +#define ADIN1300_MII_EXT_REG_PTR 0x0010
> +#define ADIN1300_MII_EXT_REG_DATA 0x0011
> +
> #define ADIN1300_INT_MASK_REG 0x0018
> #define ADIN1300_INT_MDIO_SYNC_EN BIT(9)
> #define ADIN1300_INT_ANEG_STAT_CHNG_EN BIT(8)
> @@ -53,6 +56,45 @@ static int adin_phy_config_intr(struct phy_device *phydev)
> ADIN1300_INT_MASK_EN);
> }
>
> +static int adin_read_mmd(struct phy_device *phydev, int devad, u16 regnum)
> +{
> + struct mii_bus *bus = phydev->mdio.bus;
> + int phy_addr = phydev->mdio.addr;
> + int err;
> +
> + if (phydev->is_c45) {
Similar to what we discussed regarding feature detection:
Flag is_c45 shouldn't be set with these PHY's, therefore this is dead code.
> + u32 addr = MII_ADDR_C45 | (devad << 16) | (regnum & 0xffff);
> +
> + return __mdiobus_read(bus, phy_addr, addr);
> + }
> +
> + err = __mdiobus_write(bus, phy_addr, ADIN1300_MII_EXT_REG_PTR, regnum);
> + if (err)
> + return err;
> +
> + return __mdiobus_read(bus, phy_addr, ADIN1300_MII_EXT_REG_DATA);
> +}
> +
> +static int adin_write_mmd(struct phy_device *phydev, int devad, u16 regnum,
> + u16 val)
> +{
> + struct mii_bus *bus = phydev->mdio.bus;
> + int phy_addr = phydev->mdio.addr;
> + int err;
> +
> + if (phydev->is_c45) {
> + u32 addr = MII_ADDR_C45 | (devad << 16) | (regnum & 0xffff);
> +
> + return __mdiobus_write(bus, phy_addr, addr, val);
> + }
> +
> + err = __mdiobus_write(bus, phy_addr, ADIN1300_MII_EXT_REG_PTR, regnum);
> + if (err)
> + return err;
> +
> + return __mdiobus_write(bus, phy_addr, ADIN1300_MII_EXT_REG_DATA, val);
> +}
> +
> static struct phy_driver adin_driver[] = {
> {
> PHY_ID_MATCH_MODEL(PHY_ID_ADIN1200),
> @@ -64,6 +106,8 @@ static struct phy_driver adin_driver[] = {
> .config_intr = adin_phy_config_intr,
> .resume = genphy_resume,
> .suspend = genphy_suspend,
> + .read_mmd = adin_read_mmd,
> + .write_mmd = adin_write_mmd,
> },
> {
> PHY_ID_MATCH_MODEL(PHY_ID_ADIN1300),
> @@ -75,6 +119,8 @@ static struct phy_driver adin_driver[] = {
> .config_intr = adin_phy_config_intr,
> .resume = genphy_resume,
> .suspend = genphy_suspend,
> + .read_mmd = adin_read_mmd,
> + .write_mmd = adin_write_mmd,
> },
> };
>
>
^ permalink raw reply
* Re: [PATCH v2 00/17] Networking driver debugfs cleanups
From: David Miller @ 2019-08-09 18:20 UTC (permalink / raw)
To: gregkh; +Cc: netdev
In-Reply-To: <20190809123108.27065-1-gregkh@linuxfoundation.org>
From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Date: Fri, 9 Aug 2019 14:30:51 +0200
> v2: fix up build warnings, it's as if I never even built these. Ugh, so
> sorry for wasting people's time with the v1 series. I need to stop
> relying on 0-day as it isn't working well anymore :(
One more try Greg:
drivers/net/wimax/i2400m/debugfs.c: In function ‘i2400m_debugfs_add’:
drivers/net/wimax/i2400m/debugfs.c:192:17: warning: unused variable ‘dev’ [-Wunused-variable]
struct device *dev = i2400m_dev(i2400m);
^~~
^ permalink raw reply
* Re: [PATCH v2 09/13] net: lpc-enet: fix printk format strings
From: Arnd Bergmann @ 2019-08-09 18:20 UTC (permalink / raw)
To: Joe Perches
Cc: soc, Vladimir Zapolskiy, Sylvain Lemieux, Linux ARM,
Linux Kernel Mailing List, kbuild test robot, David S. Miller,
Networking
In-Reply-To: <dc0de0cd9a1e24477b20d563199e800b98d933f6.camel@perches.com>
On Fri, Aug 9, 2019 at 6:30 PM Joe Perches <joe@perches.com> wrote:
> > @@ -1333,13 +1333,14 @@ static int lpc_eth_drv_probe(struct platform_device *pdev)
> > pldat->dma_buff_base_p = dma_handle;
> >
> > netdev_dbg(ndev, "IO address space :%pR\n", res);
> > - netdev_dbg(ndev, "IO address size :%d\n", resource_size(res));
> > + netdev_dbg(ndev, "IO address size :%zd\n",
> > + (size_t)resource_size(res));
>
> Ideally all these would use %zu not %zd
Ok, changed now.
Arnd
^ permalink raw reply
* Re: [PATCH net-next 00/10] drop_monitor: Capture dropped packets and metadata
From: Toke Høiland-Jørgensen @ 2019-08-09 18:15 UTC (permalink / raw)
To: Ido Schimmel
Cc: netdev, davem, nhorman, jiri, dsahern, roopa, nikolay,
jakub.kicinski, andy, f.fainelli, andrew, vivien.didelot, mlxsw,
Ido Schimmel
In-Reply-To: <20190809125418.GB2931@splinter>
Ido Schimmel <idosch@idosch.org> writes:
> On Fri, Aug 09, 2019 at 10:41:47AM +0200, Toke Høiland-Jørgensen wrote:
>> This is great. Are you planning to add the XDP integration as well? :)
>
> Thanks, Toke. From one of your previous replies I got the impression
> that another hook needs to be added in order to catch 'XDP_DROP' as it
> is not covered by the 'xdp_exception' tracepoint which only covers
> 'XDP_ABORTED'. If you can take care of that, I can look into the
> integration with drop monitor.
>
> I kept the XDP case in mind while working on the hardware originated
> drops and I think that extending drop monitor for this use case should
> be relatively straightforward. I will Cc you on the patchset that adds
> monitoring of hardware drops and comment with how I think XDP
> integration should look like.
SGTM, thanks!
-Toke
^ permalink raw reply
* Re: KASAN: null-ptr-deref Write in rxrpc_unuse_local
From: syzbot @ 2019-08-09 18:09 UTC (permalink / raw)
To: davem, dhowells, linux-afs, linux-kernel, netdev, syzkaller-bugs
In-Reply-To: <0000000000008cf14e058fad0c41@google.com>
syzbot has bisected this bug to:
commit 5c2833938bf50d502586e16b9dad1e3cf88fda6f
Author: David Howells <dhowells@redhat.com>
Date: Wed Jul 31 15:26:05 2019 +0000
rxrpc: Fix local endpoint refcounting
bisection log: https://syzkaller.appspot.com/x/bisect.txt?x=1519a11c600000
start commit: 87b983f5 Add linux-next specific files for 20190809
git tree: linux-next
final crash: https://syzkaller.appspot.com/x/report.txt?x=1719a11c600000
console output: https://syzkaller.appspot.com/x/log.txt?x=1319a11c600000
kernel config: https://syzkaller.appspot.com/x/.config?x=28eea330e11df0eb
dashboard link: https://syzkaller.appspot.com/bug?extid=20dee719a2e090427b5f
syz repro: https://syzkaller.appspot.com/x/repro.syz?x=17ceae36600000
C reproducer: https://syzkaller.appspot.com/x/repro.c?x=10ebc40e600000
Reported-by: syzbot+20dee719a2e090427b5f@syzkaller.appspotmail.com
Fixes: 5c2833938bf5 ("rxrpc: Fix local endpoint refcounting")
For information about bisection process see: https://goo.gl/tpsmEJ#bisection
^ permalink raw reply
* Re: [PATCH net-next] tcp: batch calls to sk_flush_backlog()
From: David Miller @ 2019-08-09 18:07 UTC (permalink / raw)
To: edumazet; +Cc: netdev, eric.dumazet, soheil
In-Reply-To: <20190809120447.93591-1-edumazet@google.com>
From: Eric Dumazet <edumazet@google.com>
Date: Fri, 9 Aug 2019 05:04:47 -0700
> Starting from commit d41a69f1d390 ("tcp: make tcp_sendmsg() aware of socket backlog")
> loopback flows got hurt, because for each skb sent, the socket receives an
> immediate ACK and sk_flush_backlog() causes extra work.
>
> Intent was to not let the backlog grow too much, but we went a bit too far.
>
> We can check the backlog every 16 skbs (about 1MB chunks)
> to increase TCP over loopback performance by about 15 %
>
> Note that the call to sk_flush_backlog() handles a single ACK,
> thanks to coalescing done on backlog, but cleans the 16 skbs
> found in rtx rb-tree.
>
> Reported-by: Soheil Hassas Yeganeh <soheil@google.com>
> Signed-off-by: Eric Dumazet <edumazet@google.com>
Applied.
^ permalink raw reply
* Re: [PATCH v3] tools: bpftool: fix reading from /proc/config.gz
From: Quentin Monnet @ 2019-08-09 17:44 UTC (permalink / raw)
To: Peter Wu, Alexei Starovoitov, Daniel Borkmann
Cc: netdev, Stanislav Fomichev, Jakub Kicinski
In-Reply-To: <20190809003911.7852-1-peter@lekensteyn.nl>
2019-08-09 01:39 UTC+0100 ~ Peter Wu <peter@lekensteyn.nl>
> /proc/config has never existed as far as I can see, but /proc/config.gz
> is present on Arch Linux. Add support for decompressing config.gz using
> zlib which is a mandatory dependency of libelf. Replace existing stdio
> functions with gzFile operations since the latter transparently handles
> uncompressed and gzip-compressed files.
>
> Cc: Quentin Monnet <quentin.monnet@netronome.com>
> Signed-off-by: Peter Wu <peter@lekensteyn.nl>
> ---
> v3: replace popen(gunzip) by linking directly to zlib. Reword commit
> message, remove "Fixes" line. (this patch)
> v2: fix style (reorder vars as reverse xmas tree, rename function,
> braces), fallback to /proc/config.gz if uname() fails.
> https://lkml.kernel.org/r/20190806010702.3303-1-peter@lekensteyn.nl
> v1: https://lkml.kernel.org/r/20190805001541.8096-1-peter@lekensteyn.nl
>
> Hi,
>
> Thanks to Jakub for observing that zlib is already used by libelf, this
> simplifies the patch tremendously as the same API can be used for both
> compressed and uncompressed files. No special case exists anymore for
> fclose/pclose.
>
> According to configure.ac in elfutils, zlib is mandatory, so I just
> assume it to be available. For simplicity I also silently assume lines
> to be less than 4096 characters. If that is not the case, then lines
> will appear truncated, but that should not be an issue for the
> CONFIG_xyz lines that we are scanning for.
>
> Jakub requested the handle leak fix to be posted separately against the
> bpf tree, but since the whole code is rewritten I am not sure if it is
> worth it. It is an unusual edge case: /boot/config-$(uname -r) could be
> opened, but starts with unexpected data.
>
> Kind regards,
> Peter
This version seems good to me, thanks!
Reviewed-by: Quentin Monnet <quentin.monnet@netronome.com>
^ permalink raw reply
* Re: [PATCH v3 net-next 0/3] net: batched receive in GRO path
From: Edward Cree @ 2019-08-09 17:32 UTC (permalink / raw)
To: Ioana Ciocoi Radulescu
Cc: David Miller, netdev, Eric Dumazet,
linux-net-drivers@solarflare.com
In-Reply-To: <AM0PR04MB4994C1A8F32FB6C9A7EE057E94D60@AM0PR04MB4994.eurprd04.prod.outlook.com>
On 09/08/2019 18:14, Ioana Ciocoi Radulescu wrote:
> Hi Edward,
>
> I'm probably missing a lot of context here, but is there a reason
> this change targets only the napi_gro_frags() path and not the
> napi_gro_receive() one?
> I'm trying to understand what drivers that don't call napi_gro_frags()
> should do in order to benefit from this batching feature.
The sfc driver (which is what I have lots of hardware for, so I can
test it) uses napi_gro_frags().
It should be possible to do a similar patch to napi_gro_receive(),
if someone wants to put in the effort of writing and testing it.
However, there are many more callers, so more effort required to
make sure none of them care whether the return value is GRO_DROP
or GRO_NORMAL (since the listified version cannot give that
indication).
Also, the guidance from Eric is that drivers seeking high performance
should use napi_gro_frags(), as this allows GRO to recycle the SKB.
All of this together means I don't plan to submit such a patch; I
would however be happy to review a patch if someone else writes one.
HTH,
-Ed
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox