Re: [PATCH net-next v5 1/7] bnxt_en: add support for rx-copybreak ethtool command

[Andy Gospodarek <andrew.gospodarek@broadcom.com>]

On Wed, Nov 13, 2024 at 05:32:15PM +0000, Taehee Yoo wrote:
> The bnxt_en driver supports rx-copybreak, but it couldn't be set by
> userspace. Only the default value(256) has worked.
> This patch makes the bnxt_en driver support following command.
> `ethtool --set-tunable <devname> rx-copybreak <value> ` and
> `ethtool --get-tunable <devname> rx-copybreak`.
> 
> By this patch, hds_threshol is set to the rx-copybreak value.
> But it will be set by `ethtool -G eth0 header-data-split-thresh N`
> in the next patch.
> 
> Reviewed-by: Brett Creeley <brett.creeley@amd.com>
> Tested-by: Stanislav Fomichev <sdf@fomichev.me>
> Signed-off-by: Taehee Yoo <ap420073@gmail.com>

Tested-by: Andy Gospodarek <gospo@broadcom.com>

> ---
> 
> v5:
>  - Do not set HDS if XDP is attached.
>  - rx_size and pkt_size are always bigger than 256.
> 
> v4:
>  - Remove min rx-copybreak value.
>  - Add Review tag from Brett.
>  - Add Test tag from Stanislav.
> 
> v3:
>  - Update copybreak value after closing nic and before opening nic when
>    the device is running.
> 
> v2:
>  - Define max/vim rx_copybreak value.
> 
>  drivers/net/ethernet/broadcom/bnxt/bnxt.c     | 28 ++++++-----
>  drivers/net/ethernet/broadcom/bnxt/bnxt.h     |  5 +-
>  .../net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 49 ++++++++++++++++++-
>  3 files changed, 68 insertions(+), 14 deletions(-)
> 
> diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
> index 4c1302a8f72d..d521b8918c02 100644
> --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
> +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
> @@ -81,7 +81,6 @@ MODULE_DESCRIPTION("Broadcom NetXtreme network driver");
>  
>  #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
>  #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
> -#define BNXT_RX_COPY_THRESH 256
>  
>  #define BNXT_TX_PUSH_THRESH 164
>  
> @@ -1328,13 +1327,13 @@ static struct sk_buff *bnxt_copy_data(struct bnxt_napi *bnapi, u8 *data,
>  	if (!skb)
>  		return NULL;
>  
> -	dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
> +	dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copybreak,
>  				bp->rx_dir);
>  
>  	memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
>  	       len + NET_IP_ALIGN);
>  
> -	dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
> +	dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copybreak,
>  				   bp->rx_dir);
>  
>  	skb_put(skb, len);
> @@ -1827,7 +1826,7 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
>  		return NULL;
>  	}
>  
> -	if (len <= bp->rx_copy_thresh) {
> +	if (len <= bp->rx_copybreak) {
>  		skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
>  		if (!skb) {
>  			bnxt_abort_tpa(cpr, idx, agg_bufs);
> @@ -2161,7 +2160,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
>  		}
>  	}
>  
> -	if (len <= bp->rx_copy_thresh) {
> +	if (len <= bp->rx_copybreak) {
>  		if (!xdp_active)
>  			skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
>  		else
> @@ -4452,6 +4451,11 @@ void bnxt_set_tpa_flags(struct bnxt *bp)
>  		bp->flags |= BNXT_FLAG_GRO;
>  }
>  
> +static void bnxt_init_ring_params(struct bnxt *bp)
> +{
> +	bp->rx_copybreak = BNXT_DEFAULT_RX_COPYBREAK;
> +}
> +
>  /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
>   * be set on entry.
>   */
> @@ -4466,7 +4470,6 @@ void bnxt_set_ring_params(struct bnxt *bp)
>  	rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
>  		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
>  
> -	bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
>  	ring_size = bp->rx_ring_size;
>  	bp->rx_agg_ring_size = 0;
>  	bp->rx_agg_nr_pages = 0;
> @@ -4511,7 +4514,9 @@ void bnxt_set_ring_params(struct bnxt *bp)
>  				  ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
>  				  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
>  		} else {
> -			rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
> +			rx_size = SKB_DATA_ALIGN(max(BNXT_DEFAULT_RX_COPYBREAK,
> +						     bp->rx_copybreak) +
> +						 NET_IP_ALIGN);
>  			rx_space = rx_size + NET_SKB_PAD +
>  				SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
>  		}
> @@ -6417,16 +6422,14 @@ static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, struct bnxt_vnic_info *vnic)
>  
>  	req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
>  	req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
> +	req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
>  
> -	if (BNXT_RX_PAGE_MODE(bp)) {
> -		req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
> -	} else {
> +	if (!BNXT_RX_PAGE_MODE(bp) && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
>  		req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
>  					  VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
>  		req->enables |=
>  			cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
> -		req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
> -		req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
> +		req->hds_threshold = cpu_to_le16(bp->rx_copybreak);
>  	}
>  	req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
>  	return hwrm_req_send(bp, req);
> @@ -15872,6 +15875,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
>  	bnxt_init_l2_fltr_tbl(bp);
>  	bnxt_set_rx_skb_mode(bp, false);
>  	bnxt_set_tpa_flags(bp);
> +	bnxt_init_ring_params(bp);
>  	bnxt_set_ring_params(bp);
>  	bnxt_rdma_aux_device_init(bp);
>  	rc = bnxt_set_dflt_rings(bp, true);
> diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
> index 649955fa3e37..d1eef880eec5 100644
> --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
> +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
> @@ -34,6 +34,9 @@
>  #include <linux/firmware/broadcom/tee_bnxt_fw.h>
>  #endif
>  
> +#define BNXT_DEFAULT_RX_COPYBREAK 256
> +#define BNXT_MAX_RX_COPYBREAK 1024
> +
>  extern struct list_head bnxt_block_cb_list;
>  
>  struct page_pool;
> @@ -2300,7 +2303,7 @@ struct bnxt {
>  	enum dma_data_direction	rx_dir;
>  	u32			rx_ring_size;
>  	u32			rx_agg_ring_size;
> -	u32			rx_copy_thresh;
> +	u32			rx_copybreak;
>  	u32			rx_ring_mask;
>  	u32			rx_agg_ring_mask;
>  	int			rx_nr_pages;
> diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
> index cfd2c65b1c90..adf30d1f738f 100644
> --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
> +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
> @@ -4318,6 +4318,50 @@ static int bnxt_get_eee(struct net_device *dev, struct ethtool_keee *edata)
>  	return 0;
>  }
>  
> +static int bnxt_set_tunable(struct net_device *dev,
> +			    const struct ethtool_tunable *tuna,
> +			    const void *data)
> +{
> +	struct bnxt *bp = netdev_priv(dev);
> +	u32 rx_copybreak;
> +
> +	switch (tuna->id) {
> +	case ETHTOOL_RX_COPYBREAK:
> +		rx_copybreak = *(u32 *)data;
> +		if (rx_copybreak > BNXT_MAX_RX_COPYBREAK)
> +			return -ERANGE;
> +		if (rx_copybreak != bp->rx_copybreak) {
> +			if (netif_running(dev)) {
> +				bnxt_close_nic(bp, false, false);
> +				bp->rx_copybreak = rx_copybreak;
> +				bnxt_set_ring_params(bp);
> +				bnxt_open_nic(bp, false, false);
> +			} else {
> +				bp->rx_copybreak = rx_copybreak;
> +			}
> +		}
> +		return 0;
> +	default:
> +		return -EOPNOTSUPP;
> +	}
> +}
> +
> +static int bnxt_get_tunable(struct net_device *dev,
> +			    const struct ethtool_tunable *tuna, void *data)
> +{
> +	struct bnxt *bp = netdev_priv(dev);
> +
> +	switch (tuna->id) {
> +	case ETHTOOL_RX_COPYBREAK:
> +		*(u32 *)data = bp->rx_copybreak;
> +		break;
> +	default:
> +		return -EOPNOTSUPP;
> +	}
> +
> +	return 0;
> +}
> +
>  static int bnxt_read_sfp_module_eeprom_info(struct bnxt *bp, u16 i2c_addr,
>  					    u16 page_number, u8 bank,
>  					    u16 start_addr, u16 data_length,
> @@ -4768,7 +4812,8 @@ static int bnxt_run_loopback(struct bnxt *bp)
>  	cpr = &rxr->bnapi->cp_ring;
>  	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
>  		cpr = rxr->rx_cpr;
> -	pkt_size = min(bp->dev->mtu + ETH_HLEN, bp->rx_copy_thresh);
> +	pkt_size = min(bp->dev->mtu + ETH_HLEN, max(BNXT_DEFAULT_RX_COPYBREAK,
> +						    bp->rx_copybreak));
>  	skb = netdev_alloc_skb(bp->dev, pkt_size);
>  	if (!skb)
>  		return -ENOMEM;
> @@ -5341,6 +5386,8 @@ const struct ethtool_ops bnxt_ethtool_ops = {
>  	.get_link_ext_stats	= bnxt_get_link_ext_stats,
>  	.get_eee		= bnxt_get_eee,
>  	.set_eee		= bnxt_set_eee,
> +	.get_tunable		= bnxt_get_tunable,
> +	.set_tunable		= bnxt_set_tunable,
>  	.get_module_info	= bnxt_get_module_info,
>  	.get_module_eeprom	= bnxt_get_module_eeprom,
>  	.get_module_eeprom_by_page = bnxt_get_module_eeprom_by_page,
> -- 
> 2.34.1
>