Re: [Intel-wired-lan] [PATCH iwl-next v2] ice: Remove and readd netdev during devlink reload

[Wojciech Drewek <wojciech.drewek@intel.com>]

On 27.01.2024 00:19, Brett Creeley wrote:
> 
> 
> On 1/25/2024 12:54 AM, Wojciech Drewek wrote:
>> Caution: This message originated from an External Source. Use proper caution when opening attachments, clicking links, or responding.
>>
>>
>> Recent changes to the devlink reload (commit 9b2348e2d6c9
>> ("devlink: warn about existing entities during reload-reinit"))
>> force the drivers to destroy devlink ports during reinit.
>> Adjust ice driver to this requirement, unregister netdvice, destroy
>> devlink port. ice_init_eth() was removed and all the common code
>> between probe and reload was moved to ice_load().
>>
>> During devlink reload we can't take devl_lock (it's already taken)
>> and in ice_probe() we have to lock it. Use devl_* variant of the API
>> which does not acquire and release devl_lock. Guard ice_load()
>> with devl_lock only in case of probe.
>>
>> Introduce ice_debugfs_fwlog_deinit() in order to release PF's
>> debugfs entries. Move ice_debugfs_exit() call to ice_module_exit().
>>
>> Suggested-by: Jiri Pirko <jiri@nvidia.com>
>> Reviewed-by: Przemek Kitszel <przemyslaw.kitszel@intel.com>
>> Signed-off-by: Wojciech Drewek <wojciech.drewek@intel.com>
>> ---
>> v2: empty init removed in ice_devlink_reinit_up
>> ---
>>   drivers/net/ethernet/intel/ice/ice.h         |   3 +
>>   drivers/net/ethernet/intel/ice/ice_debugfs.c |  10 +
>>   drivers/net/ethernet/intel/ice/ice_devlink.c |  68 ++++++-
>>   drivers/net/ethernet/intel/ice/ice_fwlog.c   |   2 +
>>   drivers/net/ethernet/intel/ice/ice_main.c    | 189 ++++++-------------
>>   5 files changed, 139 insertions(+), 133 deletions(-)
>>
>> diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
>> index e841f6c4f1c4..39734e5b9d56 100644
>> --- a/drivers/net/ethernet/intel/ice/ice.h
>> +++ b/drivers/net/ethernet/intel/ice/ice.h
>> @@ -897,6 +897,7 @@ static inline bool ice_is_adq_active(struct ice_pf *pf)
>>   }
>>
>>   void ice_debugfs_fwlog_init(struct ice_pf *pf);
>> +void ice_debugfs_fwlog_deinit(struct ice_pf *pf);
>>   void ice_debugfs_init(void);
>>   void ice_debugfs_exit(void);
>>   void ice_pf_fwlog_update_module(struct ice_pf *pf, int log_level, int module);
>> @@ -984,6 +985,8 @@ void ice_service_task_schedule(struct ice_pf *pf);
>>   int ice_load(struct ice_pf *pf);
>>   void ice_unload(struct ice_pf *pf);
>>   void ice_adv_lnk_speed_maps_init(void);
>> +int ice_init_dev(struct ice_pf *pf);
>> +void ice_deinit_dev(struct ice_pf *pf);
>>
>>   /**
>>    * ice_set_rdma_cap - enable RDMA support
>> diff --git a/drivers/net/ethernet/intel/ice/ice_debugfs.c b/drivers/net/ethernet/intel/ice/ice_debugfs.c
>> index c2bfba6b9ead..8fdcdfb804b3 100644
>> --- a/drivers/net/ethernet/intel/ice/ice_debugfs.c
>> +++ b/drivers/net/ethernet/intel/ice/ice_debugfs.c
>> @@ -647,6 +647,16 @@ void ice_debugfs_fwlog_init(struct ice_pf *pf)
>>          kfree(fw_modules);
>>   }
>>
>> +/**
>> + * ice_debugfs_fwlog_deinit - cleanup PF's debugfs
>> + * @pf: pointer to the PF struct
>> + */
>> +void ice_debugfs_fwlog_deinit(struct ice_pf *pf)
>> +{
>> +       debugfs_remove_recursive(pf->ice_debugfs_pf);
>> +       pf->ice_debugfs_pf = NULL;
>> +}
> 
> This function seems misleading to me. The ice_pf structure has the following debugfs dentry pointers:
> 
> struct dentry *ice_debugfs_pf;
> struct dentry *ice_debugfs_pf_fwlog;
> struct dentry *ice_debugfs_pf_fwlog_modules;
> 
> The function name is ice_debugfs_fwlog_deinit(), however it seems you are removing debugfs entries recursively from ice_debugfs_pf.
> 
> Maybe the function should be called:
> 
> ice_debugfs_deinit()?

ice_debugfs_pf_deinit() is even better I think since it removes debugfs entries per PF

> 
> Also, I know your commit didn't introduce the naming scheme for the debugfs members in struct ice_pf, but it's a bit strange having "ice" or "pf" in their name. It might be worth a follow up to fix these names to something like the following:
> 
> struct dentry *debugfs;
> struct dentry *debugfs_fwlog;
> struct dentry *debugfs_fwlog_modules;

We will do the follow up on that.

> 
> Thanks,
> 
> Brett
>> +
>>   /**
>>    * ice_debugfs_init - create root directory for debugfs entries
>>    */
>> diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
>> index 97182bacafa3..a428e24439d0 100644
>> --- a/drivers/net/ethernet/intel/ice/ice_devlink.c
>> +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
>> @@ -444,6 +444,20 @@ ice_devlink_reload_empr_start(struct ice_pf *pf,
>>          return 0;
>>   }
>>
>> +/**
>> + * ice_devlink_reinit_down - unload given PF
>> + * @pf: pointer to the PF struct
>> + */
>> +static void ice_devlink_reinit_down(struct ice_pf *pf)
>> +{
>> +       /* No need to take devl_lock, it's already taken by devlink API */
>> +       ice_unload(pf);
>> +       rtnl_lock();
>> +       ice_vsi_decfg(ice_get_main_vsi(pf));
>> +       rtnl_unlock();
>> +       ice_deinit_dev(pf);
>> +}
>> +
>>   /**
>>    * ice_devlink_reload_down - prepare for reload
>>    * @devlink: pointer to the devlink instance to reload
>> @@ -477,7 +491,7 @@ ice_devlink_reload_down(struct devlink *devlink, bool netns_change,
>>                                             "Remove all VFs before doing reinit\n");
>>                          return -EOPNOTSUPP;
>>                  }
>> -               ice_unload(pf);
>> +               ice_devlink_reinit_down(pf);
>>                  return 0;
>>          case DEVLINK_RELOAD_ACTION_FW_ACTIVATE:
>>                  return ice_devlink_reload_empr_start(pf, extack);
>> @@ -1269,6 +1283,45 @@ static int ice_devlink_set_parent(struct devlink_rate *devlink_rate,
>>          return status;
>>   }
>>
>> +/**
>> + * ice_devlink_reinit_up - do reinit of the given PF
>> + * @pf: pointer to the PF struct
>> + */
>> +static int ice_devlink_reinit_up(struct ice_pf *pf)
>> +{
>> +       struct ice_vsi *vsi = ice_get_main_vsi(pf);
>> +       struct ice_vsi_cfg_params params;
>> +       int err;
>> +
>> +       err = ice_init_dev(pf);
>> +       if (err)
>> +               return err;
>> +
>> +       params = ice_vsi_to_params(vsi);
>> +       params.flags = ICE_VSI_FLAG_INIT;
>> +
>> +       rtnl_lock();
>> +       err = ice_vsi_cfg(vsi, &params);
>> +       if (err)
>> +               goto err_vsi_cfg;
>> +       rtnl_unlock();
> 
> Maybe just personal opinion, but this locking seems a bit confusing to me. I think it might be more readable as:
> 
> rtnl_lock();
> err = ice_vsi_cfg(vsi, &params);
> rtnl_unlock();
> if (err)
>     goto err_vsi_cfg;
> 
> Then below...

Agree, I'll fix that in the next version

>> +
>> +       /* No need to take devl_lock, it's already taken by devlink API */
>> +       err = ice_load(pf);
>> +       if (err)
>> +               goto err_load;
>> +
>> +       return 0;
>> +
>> +err_load:
>> +       rtnl_lock();
>> +       ice_vsi_decfg(vsi);
>> +err_vsi_cfg:
>> +       rtnl_unlock();
>> +       ice_deinit_dev(pf);
>> +       return err;
> 
> err_load:
>     rtnl_lock();
>     ice_vsi_decfg(vsi);
>     rtnl_unlock();
> err_vsi_cfg:
>     ice_deinit_dev(pf);
>     return err;
>> +}
>> +
>>   /**
>>    * ice_devlink_reload_up - do reload up after reinit
>>    * @devlink: pointer to the devlink instance reloading
>> @@ -1289,7 +1342,7 @@ ice_devlink_reload_up(struct devlink *devlink,
>>          switch (action) {
>>          case DEVLINK_RELOAD_ACTION_DRIVER_REINIT:
>>                  *actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT);
>> -               return ice_load(pf);
>> +               return ice_devlink_reinit_up(pf);
>>          case DEVLINK_RELOAD_ACTION_FW_ACTIVATE:
>>                  *actions_performed = BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE);
>>                  return ice_devlink_reload_empr_finish(pf, extack);
>> @@ -1695,6 +1748,7 @@ static const struct devlink_port_ops ice_devlink_port_ops = {
>>    * @pf: the PF to create a devlink port for
>>    *
>>    * Create and register a devlink_port for this PF.
>> + * This function has to be called under devl_lock.
>>    *
>>    * Return: zero on success or an error code on failure.
>>    */

<...>